Message ID | 20190717134335.15351-23-alex.bennee@linaro.org |
---|---|
State | Superseded |
Headers | show |
Series | testing/next for 4.1-rc2 (win, travis, iotests) | expand |
On 7/17/19 3:43 PM, Alex Bennée wrote: > From: Thomas Huth <thuth@redhat.com> > > People often forget to run the iotests before submitting patches or pull > requests - this is likely due to the fact that we do not run the tests > during our mandatory "make check" tests yet. Now that we've got a proper > "auto" group of iotests that should be fine to run in every environment, > we can enable the iotests during "make check" again by running the "auto" > tests by default from the check-block.sh script. > > Some cases still need to be checked first, though: iotests need bash and > GNU sed (otherwise they fail), and if gprof is enabled, it spoils the > output of some test cases causing them to fail. So if we detect that one > of the required programs is missing or that gprof is enabled, we still > have to skip the iotests to avoid failures. > > And finally, since we are using check-block.sh now again, this patch also > removes the qemu-iotests-quick.sh script since we do not need that anymore > (and having two shell wrapper scripts around the block tests seems rather > confusing than helpful). > > Signed-off-by: Thomas Huth <thuth@redhat.com> > [AJB: -makecheck to check-block.sh, move check-block to start and gate it] > Signed-off-by: Alex Bennée <alex.bennee@linaro.org> > Message-Id: <20190717111947.30356-4-thuth@redhat.com> > Signed-off-by: Alex Bennée <alex.bennee@linaro.org> > --- > tests/Makefile.include | 10 +++++---- > tests/check-block.sh | 44 ++++++++++++++++++++++++++++--------- > tests/qemu-iotests-quick.sh | 8 ------- > 3 files changed, 40 insertions(+), 22 deletions(-) > delete mode 100755 tests/qemu-iotests-quick.sh > > diff --git a/tests/Makefile.include b/tests/Makefile.include > index fd7fdb86586..6f02dfcc019 100644 > --- a/tests/Makefile.include > +++ b/tests/Makefile.include > @@ -142,7 +142,7 @@ check-unit-y += tests/test-uuid$(EXESUF) > check-unit-y += tests/ptimer-test$(EXESUF) > check-unit-y += tests/test-qapi-util$(EXESUF) > > -check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh > +check-block-$(call land,$(CONFIG_POSIX),$(CONFIG_SOFTMMU)) += tests/check-block.sh > > # All QTests for now are POSIX-only, but the dependencies are > # really in libqtest, not in the testcases themselves. > @@ -1092,8 +1092,10 @@ clean-tcg: $(CLEAN_TCG_TARGET_RULES) > > QEMU_IOTESTS_HELPERS-$(call land,$(CONFIG_SOFTMMU),$(CONFIG_LINUX)) = tests/qemu-iotests/socket_scm_helper$(EXESUF) > > -.PHONY: check-tests/qemu-iotests-quick.sh > -check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF) qemu-io$(EXESUF) qemu-nbd$(EXESUF) $(QEMU_IOTESTS_HELPERS-y) > +.PHONY: check-tests/check-block.sh > +check-tests/check-block.sh: tests/check-block.sh qemu-img$(EXESUF) \ > + qemu-io$(EXESUF) qemu-nbd$(EXESUF) $(QEMU_IOTESTS_HELPERS-y) \ > + $(patsubst %,%/all,$(filter %-softmmu,$(TARGET_DIRS))) > $< > > .PHONY: $(patsubst %, check-%, $(check-qapi-schema-y)) > @@ -1167,7 +1169,7 @@ check-acceptance: check-venv $(TESTS_RESULTS_DIR) > check-qapi-schema: $(patsubst %,check-%, $(check-qapi-schema-y)) check-tests/qapi-schema/doc-good.texi > check-qtest: $(patsubst %,check-qtest-%, $(QTEST_TARGETS)) > check-block: $(patsubst %,check-%, $(check-block-y)) > -check: check-qapi-schema check-unit check-softfloat check-qtest check-decodetree > +check: check-block check-qapi-schema check-unit check-softfloat check-qtest check-decodetree > check-clean: > rm -rf $(check-unit-y) tests/*.o $(QEMU_IOTESTS_HELPERS-y) > rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y)) $(check-qtest-generic-y)) > diff --git a/tests/check-block.sh b/tests/check-block.sh > index f3d12fd602d..c8b6cec3f68 100755 > --- a/tests/check-block.sh > +++ b/tests/check-block.sh > @@ -1,24 +1,48 @@ > #!/bin/sh > > -FORMAT_LIST="raw qcow2 qed vmdk vpc" > +# Honor the SPEED environment variable, just like we do it for the qtests. > +if [ "$SPEED" = "slow" ]; then > + format_list="raw qcow2" > + group= > +elif [ "$SPEED" = "thorough" ]; then > + format_list="raw qcow2 qed vmdk vpc" > + group= > +else > + format_list=qcow2 > + group="-g auto" > +fi > + > if [ "$#" -ne 0 ]; then > - FORMAT_LIST="$@" > + format_list="$@" > +fi > + > +if grep -q "TARGET_GPROF=y" *-softmmu/config-target.mak 2>/dev/null ; then > + echo "GPROF is enabled ==> Not running the qemu-iotests." > + exit 0 > fi > > -export QEMU_PROG="$PWD/x86_64-softmmu/qemu-system-x86_64" > -export QEMU_IMG_PROG="$PWD/qemu-img" > -export QEMU_IO_PROG="$PWD/qemu-io" > +if [ -z "$(find . -name 'qemu-system-*' -print)" ]; then > + echo "No qemu-system binary available ==> Not running the qemu-iotests." > + exit 0 > +fi > + > +if ! command -v bash >/dev/null 2>&1 ; then > + echo "bash not available ==> Not running the qemu-iotests." > + exit 0 > +fi > > -if [ ! -x $QEMU_PROG ]; then > - echo "'make check-block' requires qemu-system-x86_64" > - exit 1 > +if ! (sed --version | grep 'GNU sed') > /dev/null 2>&1 ; then > + if ! command -v gsed >/dev/null 2>&1; then > + echo "GNU sed not available ==> Not running the qemu-iotests." > + exit 0 > + fi > fi > > cd tests/qemu-iotests > > ret=0 > -for FMT in $FORMAT_LIST ; do > - ./check -T -nocache -$FMT || ret=1 > +for fmt in $format_list ; do > + ./check -makecheck -$fmt $group || ret=1 > done > > exit $ret > diff --git a/tests/qemu-iotests-quick.sh b/tests/qemu-iotests-quick.sh > deleted file mode 100755 > index 0e554bb9724..00000000000 > --- a/tests/qemu-iotests-quick.sh > +++ /dev/null > @@ -1,8 +0,0 @@ > -#!/bin/sh > - > -cd tests/qemu-iotests > - > -ret=0 > -TEST_DIR=${TEST_DIR:-/tmp/qemu-iotests-quick-$$} ./check -T -qcow2 -g quick || ret=1 > - > -exit $ret > Since Gerd updated the OpenBSD image, do you know if we can run vm-test again? I haven't check which what got merged from this previous series: https://lists.gnu.org/archive/html/qemu-devel/2019-01/msg07527.html
On 23/07/2019 09.20, Thomas Huth wrote: > On 22/07/2019 21.53, Philippe Mathieu-Daudé wrote: > [...] >> Since Gerd updated the OpenBSD image, do you know if we can run vm-test >> again? > > I just tried it, but the OpenBSD build seems to be completely broken right now: > > $ nice make vm-build-openbsd > VM-IMAGE openbsd > ### Downloading install iso ... > --2019-07-23 08:52:46-- https://cdn.openbsd.org/pub/OpenBSD/6.5/amd64/install65.iso > Resolving cdn.openbsd.org (cdn.openbsd.org)... 151.101.38.217 > Connecting to cdn.openbsd.org (cdn.openbsd.org)|151.101.38.217|:443... connected. > HTTP request sent, awaiting response... 200 OK > Length: 407169024 (388M) [application/octet-stream] > Saving to: ‘/home/thuth/.cache/qemu-vm/download/54ac74c2128d6c2d3ede38756576fe89c08476bd.download’ > > 100%[=======================================================================>] 407.169.024 8,55MB/s in 39s > > 2019-07-23 08:53:25 (9,98 MB/s) - ‘/home/thuth/.cache/qemu-vm/download/54ac74c2128d6c2d3ede38756576fe89c08476bd.download’ saved [407169024/407169024] > > ### Preparing iso and disk image ... > Formatting '/home/thuth/.cache/qemu-vm/images/openbsd.img.tmp', fmt=qcow2 size=21474836480 cluster_size=65536 lazy_refcounts=off refcount_bits=16 > ### Booting installer ... > console: *** read timeout *** > console: waiting for: 'timezone' > console: line buffer: > > con recv: Which disk is the root disk? ('?' for details) [sd0] > > Failed to prepare guest environment > Traceback (most recent call last): > File "/home/thuth/devel/qemu/tests/vm/basevm.py", line 353, in main > return vm.build_image(args.image) > File "/home/thuth/devel/qemu/tests/vm/openbsd", line 118, in build_image > self.console_wait_send("timezone", "UTC\n") > File "/home/thuth/devel/qemu/tests/vm/basevm.py", line 253, in console_wait_send > self.console_wait(wait) > File "/home/thuth/devel/qemu/tests/vm/basevm.py", line 215, in console_wait > chars = vm.console_socket.recv(1) > socket.timeout: timed out > make: *** [/home/thuth/.cache/qemu-vm/images/openbsd.img] Error 2 > > I even tried to delete the ~/.cache/qemu-vm folder, but that also did > not help, I'm always getting that time-out now. Does it still work for > you? FWIW, it works again with this hack: diff a/tests/vm/openbsd b/tests/vm/openbsd --- a/tests/vm/openbsd +++ b/tests/vm/openbsd @@ -115,7 +115,7 @@ class OpenBSDVM(basevm.BaseVM): self.console_send("%s\n" % self.GUEST_PASS) self.console_wait_send("Allow root ssh login", "yes\n") - self.console_wait_send("timezone", "UTC\n") + self.console_send("UTC\n") self.console_wait_send("root disk", "\n") self.console_wait_send("(W)hole disk", "\n") self.console_wait_send("(A)uto layout", "\n") ¯\_(ツ)_/¯ Thomas
On 22/07/2019 21.53, Philippe Mathieu-Daudé wrote: > On 7/17/19 3:43 PM, Alex Bennée wrote: >> From: Thomas Huth <thuth@redhat.com> >> >> People often forget to run the iotests before submitting patches or pull >> requests - this is likely due to the fact that we do not run the tests >> during our mandatory "make check" tests yet. Now that we've got a proper >> "auto" group of iotests that should be fine to run in every environment, >> we can enable the iotests during "make check" again by running the "auto" >> tests by default from the check-block.sh script. >> >> Some cases still need to be checked first, though: iotests need bash and >> GNU sed (otherwise they fail), and if gprof is enabled, it spoils the >> output of some test cases causing them to fail. So if we detect that one >> of the required programs is missing or that gprof is enabled, we still >> have to skip the iotests to avoid failures. >> >> And finally, since we are using check-block.sh now again, this patch also >> removes the qemu-iotests-quick.sh script since we do not need that anymore >> (and having two shell wrapper scripts around the block tests seems rather >> confusing than helpful). >> >> Signed-off-by: Thomas Huth <thuth@redhat.com> >> [AJB: -makecheck to check-block.sh, move check-block to start and gate it] >> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> >> Message-Id: <20190717111947.30356-4-thuth@redhat.com> >> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> >> --- >> tests/Makefile.include | 10 +++++---- >> tests/check-block.sh | 44 ++++++++++++++++++++++++++++--------- >> tests/qemu-iotests-quick.sh | 8 ------- >> 3 files changed, 40 insertions(+), 22 deletions(-) >> delete mode 100755 tests/qemu-iotests-quick.sh >> >> diff --git a/tests/Makefile.include b/tests/Makefile.include >> index fd7fdb86586..6f02dfcc019 100644 >> --- a/tests/Makefile.include >> +++ b/tests/Makefile.include >> @@ -142,7 +142,7 @@ check-unit-y += tests/test-uuid$(EXESUF) >> check-unit-y += tests/ptimer-test$(EXESUF) >> check-unit-y += tests/test-qapi-util$(EXESUF) >> >> -check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh >> +check-block-$(call land,$(CONFIG_POSIX),$(CONFIG_SOFTMMU)) += tests/check-block.sh >> >> # All QTests for now are POSIX-only, but the dependencies are >> # really in libqtest, not in the testcases themselves. >> @@ -1092,8 +1092,10 @@ clean-tcg: $(CLEAN_TCG_TARGET_RULES) >> >> QEMU_IOTESTS_HELPERS-$(call land,$(CONFIG_SOFTMMU),$(CONFIG_LINUX)) = tests/qemu-iotests/socket_scm_helper$(EXESUF) >> >> -.PHONY: check-tests/qemu-iotests-quick.sh >> -check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF) qemu-io$(EXESUF) qemu-nbd$(EXESUF) $(QEMU_IOTESTS_HELPERS-y) >> +.PHONY: check-tests/check-block.sh >> +check-tests/check-block.sh: tests/check-block.sh qemu-img$(EXESUF) \ >> + qemu-io$(EXESUF) qemu-nbd$(EXESUF) $(QEMU_IOTESTS_HELPERS-y) \ >> + $(patsubst %,%/all,$(filter %-softmmu,$(TARGET_DIRS))) >> $< >> >> .PHONY: $(patsubst %, check-%, $(check-qapi-schema-y)) >> @@ -1167,7 +1169,7 @@ check-acceptance: check-venv $(TESTS_RESULTS_DIR) >> check-qapi-schema: $(patsubst %,check-%, $(check-qapi-schema-y)) check-tests/qapi-schema/doc-good.texi >> check-qtest: $(patsubst %,check-qtest-%, $(QTEST_TARGETS)) >> check-block: $(patsubst %,check-%, $(check-block-y)) >> -check: check-qapi-schema check-unit check-softfloat check-qtest check-decodetree >> +check: check-block check-qapi-schema check-unit check-softfloat check-qtest check-decodetree >> check-clean: >> rm -rf $(check-unit-y) tests/*.o $(QEMU_IOTESTS_HELPERS-y) >> rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y)) $(check-qtest-generic-y)) >> diff --git a/tests/check-block.sh b/tests/check-block.sh >> index f3d12fd602d..c8b6cec3f68 100755 >> --- a/tests/check-block.sh >> +++ b/tests/check-block.sh >> @@ -1,24 +1,48 @@ >> #!/bin/sh >> >> -FORMAT_LIST="raw qcow2 qed vmdk vpc" >> +# Honor the SPEED environment variable, just like we do it for the qtests. >> +if [ "$SPEED" = "slow" ]; then >> + format_list="raw qcow2" >> + group= >> +elif [ "$SPEED" = "thorough" ]; then >> + format_list="raw qcow2 qed vmdk vpc" >> + group= >> +else >> + format_list=qcow2 >> + group="-g auto" >> +fi >> + >> if [ "$#" -ne 0 ]; then >> - FORMAT_LIST="$@" >> + format_list="$@" >> +fi >> + >> +if grep -q "TARGET_GPROF=y" *-softmmu/config-target.mak 2>/dev/null ; then >> + echo "GPROF is enabled ==> Not running the qemu-iotests." >> + exit 0 >> fi >> >> -export QEMU_PROG="$PWD/x86_64-softmmu/qemu-system-x86_64" >> -export QEMU_IMG_PROG="$PWD/qemu-img" >> -export QEMU_IO_PROG="$PWD/qemu-io" >> +if [ -z "$(find . -name 'qemu-system-*' -print)" ]; then >> + echo "No qemu-system binary available ==> Not running the qemu-iotests." >> + exit 0 >> +fi >> + >> +if ! command -v bash >/dev/null 2>&1 ; then >> + echo "bash not available ==> Not running the qemu-iotests." >> + exit 0 >> +fi >> >> -if [ ! -x $QEMU_PROG ]; then >> - echo "'make check-block' requires qemu-system-x86_64" >> - exit 1 >> +if ! (sed --version | grep 'GNU sed') > /dev/null 2>&1 ; then >> + if ! command -v gsed >/dev/null 2>&1; then >> + echo "GNU sed not available ==> Not running the qemu-iotests." >> + exit 0 >> + fi >> fi >> >> cd tests/qemu-iotests >> >> ret=0 >> -for FMT in $FORMAT_LIST ; do >> - ./check -T -nocache -$FMT || ret=1 >> +for fmt in $format_list ; do >> + ./check -makecheck -$fmt $group || ret=1 >> done >> >> exit $ret >> diff --git a/tests/qemu-iotests-quick.sh b/tests/qemu-iotests-quick.sh >> deleted file mode 100755 >> index 0e554bb9724..00000000000 >> --- a/tests/qemu-iotests-quick.sh >> +++ /dev/null >> @@ -1,8 +0,0 @@ >> -#!/bin/sh >> - >> -cd tests/qemu-iotests >> - >> -ret=0 >> -TEST_DIR=${TEST_DIR:-/tmp/qemu-iotests-quick-$$} ./check -T -qcow2 -g quick || ret=1 >> - >> -exit $ret >> > > Since Gerd updated the OpenBSD image, do you know if we can run vm-test > again? Oh well, looks like many iotests are failing since the "seq" program is missing: Failures: 007 011 017 020 022 025 032 035 037 046 Seems like "gseq" is available in the "coreutils" package, but only with the different name... Maybe we should rather replace the "for i in `seq ...`" loops with "for ((i=...))", especially since we require bash in the tests anyway? Thomas
On Tue, Jul 23, 2019 at 09:20:43AM +0200, Thomas Huth wrote: > On 22/07/2019 21.53, Philippe Mathieu-Daudé wrote: > [...] > > Since Gerd updated the OpenBSD image, do you know if we can run vm-test > > again? > > I just tried it, but the OpenBSD build seems to be completely broken right now: > > $ nice make vm-build-openbsd Works fine here. Can you try again with "V=1" ? thanks, Gerd
On 8/2/19 4:28 PM, Gerd Hoffmann wrote: > On Tue, Jul 23, 2019 at 09:20:43AM +0200, Thomas Huth wrote: >> On 22/07/2019 21.53, Philippe Mathieu-Daudé wrote: >> [...] >>> Since Gerd updated the OpenBSD image, do you know if we can run vm-test >>> again? >> >> I just tried it, but the OpenBSD build seems to be completely broken right now: >> >> $ nice make vm-build-openbsd > > Works fine here. > Can you try again with "V=1" ? Meanwhile, I've updated my system from RHEL7 to RHEL8, and now when I tried again, it's also working fine here. Must have been some oddity with RHEL7, I guess... Anyway, thanks for checking, Thomas
diff --git a/tests/Makefile.include b/tests/Makefile.include index fd7fdb86586..6f02dfcc019 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -142,7 +142,7 @@ check-unit-y += tests/test-uuid$(EXESUF) check-unit-y += tests/ptimer-test$(EXESUF) check-unit-y += tests/test-qapi-util$(EXESUF) -check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh +check-block-$(call land,$(CONFIG_POSIX),$(CONFIG_SOFTMMU)) += tests/check-block.sh # All QTests for now are POSIX-only, but the dependencies are # really in libqtest, not in the testcases themselves. @@ -1092,8 +1092,10 @@ clean-tcg: $(CLEAN_TCG_TARGET_RULES) QEMU_IOTESTS_HELPERS-$(call land,$(CONFIG_SOFTMMU),$(CONFIG_LINUX)) = tests/qemu-iotests/socket_scm_helper$(EXESUF) -.PHONY: check-tests/qemu-iotests-quick.sh -check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF) qemu-io$(EXESUF) qemu-nbd$(EXESUF) $(QEMU_IOTESTS_HELPERS-y) +.PHONY: check-tests/check-block.sh +check-tests/check-block.sh: tests/check-block.sh qemu-img$(EXESUF) \ + qemu-io$(EXESUF) qemu-nbd$(EXESUF) $(QEMU_IOTESTS_HELPERS-y) \ + $(patsubst %,%/all,$(filter %-softmmu,$(TARGET_DIRS))) $< .PHONY: $(patsubst %, check-%, $(check-qapi-schema-y)) @@ -1167,7 +1169,7 @@ check-acceptance: check-venv $(TESTS_RESULTS_DIR) check-qapi-schema: $(patsubst %,check-%, $(check-qapi-schema-y)) check-tests/qapi-schema/doc-good.texi check-qtest: $(patsubst %,check-qtest-%, $(QTEST_TARGETS)) check-block: $(patsubst %,check-%, $(check-block-y)) -check: check-qapi-schema check-unit check-softfloat check-qtest check-decodetree +check: check-block check-qapi-schema check-unit check-softfloat check-qtest check-decodetree check-clean: rm -rf $(check-unit-y) tests/*.o $(QEMU_IOTESTS_HELPERS-y) rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y)) $(check-qtest-generic-y)) diff --git a/tests/check-block.sh b/tests/check-block.sh index f3d12fd602d..c8b6cec3f68 100755 --- a/tests/check-block.sh +++ b/tests/check-block.sh @@ -1,24 +1,48 @@ #!/bin/sh -FORMAT_LIST="raw qcow2 qed vmdk vpc" +# Honor the SPEED environment variable, just like we do it for the qtests. +if [ "$SPEED" = "slow" ]; then + format_list="raw qcow2" + group= +elif [ "$SPEED" = "thorough" ]; then + format_list="raw qcow2 qed vmdk vpc" + group= +else + format_list=qcow2 + group="-g auto" +fi + if [ "$#" -ne 0 ]; then - FORMAT_LIST="$@" + format_list="$@" +fi + +if grep -q "TARGET_GPROF=y" *-softmmu/config-target.mak 2>/dev/null ; then + echo "GPROF is enabled ==> Not running the qemu-iotests." + exit 0 fi -export QEMU_PROG="$PWD/x86_64-softmmu/qemu-system-x86_64" -export QEMU_IMG_PROG="$PWD/qemu-img" -export QEMU_IO_PROG="$PWD/qemu-io" +if [ -z "$(find . -name 'qemu-system-*' -print)" ]; then + echo "No qemu-system binary available ==> Not running the qemu-iotests." + exit 0 +fi + +if ! command -v bash >/dev/null 2>&1 ; then + echo "bash not available ==> Not running the qemu-iotests." + exit 0 +fi -if [ ! -x $QEMU_PROG ]; then - echo "'make check-block' requires qemu-system-x86_64" - exit 1 +if ! (sed --version | grep 'GNU sed') > /dev/null 2>&1 ; then + if ! command -v gsed >/dev/null 2>&1; then + echo "GNU sed not available ==> Not running the qemu-iotests." + exit 0 + fi fi cd tests/qemu-iotests ret=0 -for FMT in $FORMAT_LIST ; do - ./check -T -nocache -$FMT || ret=1 +for fmt in $format_list ; do + ./check -makecheck -$fmt $group || ret=1 done exit $ret diff --git a/tests/qemu-iotests-quick.sh b/tests/qemu-iotests-quick.sh deleted file mode 100755 index 0e554bb9724..00000000000 --- a/tests/qemu-iotests-quick.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -cd tests/qemu-iotests - -ret=0 -TEST_DIR=${TEST_DIR:-/tmp/qemu-iotests-quick-$$} ./check -T -qcow2 -g quick || ret=1 - -exit $ret