diff mbox series

[PULL,22/22] Revert "tests/avocado: remove skips from replay_kernel"

Message ID 20240112110435.3801068-23-alex.bennee@linaro.org
State New
Headers show
Series [PULL,01/22] tests/avocado: Add a test for a little-endian microblaze machine | expand

Commit Message

Alex Bennée Jan. 12, 2024, 11:04 a.m. UTC 
This reverts commit c2ef5ee89d76f0ab77c4dd6a1c9eeed4d35d20ed.

While the fixes for #2010 and #2013 have improved things locally it
seems GitLab still continues to be flaky.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

Comments

Peter Maydell Jan. 12, 2024, 11:13 a.m. UTC | #1 
On Fri, 12 Jan 2024 at 11:11, Alex Bennée <alex.bennee@linaro.org> wrote:
>
> This reverts commit c2ef5ee89d76f0ab77c4dd6a1c9eeed4d35d20ed.
>
> While the fixes for #2010 and #2013 have improved things locally it
> seems GitLab still continues to be flaky.
>
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>
> diff --git a/tests/avocado/replay_kernel.py b/tests/avocado/replay_kernel.py
> index 6fdcbd6ac3d..1eaa36444cb 100644
> --- a/tests/avocado/replay_kernel.py
> +++ b/tests/avocado/replay_kernel.py
> @@ -98,10 +98,13 @@ def test_i386_pc(self):
>
>          self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
>
> +    # See https://gitlab.com/qemu-project/qemu/-/issues/2010
> +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test sometimes gets stuck')
>      def test_x86_64_pc(self):
>          """
>          :avocado: tags=arch:x86_64
>          :avocado: tags=machine:pc
> +        :avocado: tags=flaky
>          """
>          kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
>                        '/linux/releases/29/Everything/x86_64/os/images/pxeboot'
> @@ -132,6 +135,8 @@ def test_mips_malta(self):
>
>          self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
>
> +    # See https://gitlab.com/qemu-project/qemu/-/issues/2013
> +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
>      def test_mips64el_malta(self):
>          """
>          This test requires the ar tool to extract "data.tar.gz" from

These gitlab issues are both currently closed -- if we think the
problem is still present and are re-introducing the skip lines,
we should re-open the issues, I think.


thanks
-- PMM
Alex Bennée Jan. 12, 2024, 1:11 p.m. UTC | #2 
Peter Maydell <peter.maydell@linaro.org> writes:

> On Fri, 12 Jan 2024 at 11:11, Alex Bennée <alex.bennee@linaro.org> wrote:
>>
>> This reverts commit c2ef5ee89d76f0ab77c4dd6a1c9eeed4d35d20ed.
>>
>> While the fixes for #2010 and #2013 have improved things locally it
>> seems GitLab still continues to be flaky.
>>
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>>
>> diff --git a/tests/avocado/replay_kernel.py b/tests/avocado/replay_kernel.py
>> index 6fdcbd6ac3d..1eaa36444cb 100644
>> --- a/tests/avocado/replay_kernel.py
>> +++ b/tests/avocado/replay_kernel.py
>> @@ -98,10 +98,13 @@ def test_i386_pc(self):
>>
>>          self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
>>
>> +    # See https://gitlab.com/qemu-project/qemu/-/issues/2010
>> +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test sometimes gets stuck')
>>      def test_x86_64_pc(self):
>>          """
>>          :avocado: tags=arch:x86_64
>>          :avocado: tags=machine:pc
>> +        :avocado: tags=flaky
>>          """
>>          kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
>>                        '/linux/releases/29/Everything/x86_64/os/images/pxeboot'
>> @@ -132,6 +135,8 @@ def test_mips_malta(self):
>>
>>          self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
>>
>> +    # See https://gitlab.com/qemu-project/qemu/-/issues/2013
>> +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
>>      def test_mips64el_malta(self):
>>          """
>>          This test requires the ar tool to extract "data.tar.gz" from
>
> These gitlab issues are both currently closed -- if we think the
> problem is still present and are re-introducing the skip lines,
> we should re-open the issues, I think.

Hmm - I certainly fixed some bugs under those two. I think I'll re-spin
with a new bug raised.

>
>
> thanks
> -- PMM
diff mbox series

Patch

diff --git a/tests/avocado/replay_kernel.py b/tests/avocado/replay_kernel.py
index 6fdcbd6ac3d..1eaa36444cb 100644
--- a/tests/avocado/replay_kernel.py
+++ b/tests/avocado/replay_kernel.py
@@ -98,10 +98,13 @@  def test_i386_pc(self):
 
         self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
 
+    # See https://gitlab.com/qemu-project/qemu/-/issues/2010
+    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test sometimes gets stuck')
     def test_x86_64_pc(self):
         """
         :avocado: tags=arch:x86_64
         :avocado: tags=machine:pc
+        :avocado: tags=flaky
         """
         kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
                       '/linux/releases/29/Everything/x86_64/os/images/pxeboot'
@@ -132,6 +135,8 @@  def test_mips_malta(self):
 
         self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
 
+    # See https://gitlab.com/qemu-project/qemu/-/issues/2013
+    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
     def test_mips64el_malta(self):
         """
         This test requires the ar tool to extract "data.tar.gz" from
@@ -147,6 +152,7 @@  def test_mips64el_malta(self):
 
         :avocado: tags=arch:mips64el
         :avocado: tags=machine:malta
+        :avocado: tags=flaky
         """
         deb_url = ('http://snapshot.debian.org/archive/debian/'
                    '20130217T032700Z/pool/main/l/linux-2.6/'
@@ -194,10 +200,13 @@  def test_arm_virt(self):
 
         self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=1)
 
+    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
+
     def test_arm_cubieboard_initrd(self):
         """
         :avocado: tags=arch:arm
         :avocado: tags=machine:cubieboard
+        :avocado: tags=flaky
         """
         deb_url = ('https://apt.armbian.com/pool/main/l/'
                    'linux-5.10.16-sunxi/linux-image-current-sunxi_21.02.2_armhf.deb')
@@ -345,6 +354,7 @@  def test_m68k_mcf5208evb(self):
         file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
         self.do_test_advcal_2018(file_path, 'sanity-clause.elf')
 
+    @skip("Test currently broken") # Console stuck as of 5.2-rc1
     def test_microblaze_s3adsp1800(self):
         """
         :avocado: tags=arch:microblaze
@@ -379,6 +389,7 @@  def test_or1k_sim(self):
         file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
         self.do_test_advcal_2018(file_path, 'vmlinux')
 
+    @skip("nios2 emulation is buggy under record/replay")
     def test_nios2_10m50(self):
         """
         :avocado: tags=arch:nios2