From 100e09ad23f3a51c81fbc2524112a0537a8349fa Mon Sep 17 00:00:00 2001 From: Martin Stransky Date: Wed, 16 Dec 2020 19:32:47 +0100 Subject: Run xpcshell tests sequentially --- firefox.spec | 7 +++++-- perrors | 9 +++++++++ print_results | 7 +++++-- print_subtest | 23 ----------------------- psummary | 23 +++++++++++++++++++++++ run-tests-wayland | 4 ++-- run-tests-x11 | 4 ++-- 7 files changed, 46 insertions(+), 31 deletions(-) create mode 100755 perrors delete mode 100755 print_subtest create mode 100755 psummary diff --git a/firefox.spec b/firefox.spec index 3ed00d3..7adb53f 100644 --- a/firefox.spec +++ b/firefox.spec @@ -153,9 +153,10 @@ Source34: firefox-search-provider.ini Source35: google-loc-api-key Source37: mochitest-python.tar.gz Source38: print_results -Source39: print_subtest +Source39: perrors Source40: run-tests-x11 Source41: run-tests-wayland +Source42: psummary # Build patches Patch3: mozilla-build-arm.patch @@ -360,6 +361,7 @@ This package contains results of tests executed during build. %files -n %{testsuite_pkg_name} /%{version}%-%{release}/test_results /%{version}%-%{release}/test_summary.txt +/%{version}%-%{release}/failures-* %endif #--------------------------------------------------------------------- @@ -711,7 +713,7 @@ find-links=`pwd`/mochitest-python no-index=true EOF tar xf %{SOURCE37} -cp %{SOURCE40} %{SOURCE41} %{SOURCE38} %{SOURCE39} . +cp %{SOURCE40} %{SOURCE41} %{SOURCE42} %{SOURCE38} %{SOURCE39} . mkdir -p test_results %if %{test_on_wayland} ./run-tests-wayland @@ -856,6 +858,7 @@ sed -i -e "s/\[Crash Reporter\]/[Crash Reporter]\nEnabled=1/" %{buildroot}/%{moz %{__mkdir_p} %{buildroot}/%{version}%-%{release}/test_results %{__cp} test_results/* %{buildroot}/%{version}%-%{release}/test_results %{__cp} test_summary.txt %{buildroot}/%{version}%-%{release}/ +%{__cp} failures-*txt %{buildroot}/%{version}%-%{release}/ %endif # Default diff --git a/perrors b/perrors new file mode 100755 index 0000000..537a4d0 --- /dev/null +++ b/perrors @@ -0,0 +1,9 @@ +#!/usr/bin/bash +# Print failed tests + +TEST_DIR=$1 +TEST_FLAVOUR=$2 + +grep "TEST-UNEXPECTED-FAIL" $TEST_DIR/mochitest$TEST_FLAVOUR 2>&1 > failures-mochitest$TEST_FLAVOUR +grep --text " FAIL " $TEST_DIR/xpcshell$TEST_FLAVOUR 2>&1 > failures-xpcshell$TEST_FLAVOUR +grep --text "REFTEST TEST-UNEXPECTED-FAIL" $TEST_DIR/reftest$TEST_FLAVOUR 2>&1 > failures-reftest$TEST_FLAVOUR diff --git a/print_results b/print_results index d0bb535..0656e5f 100755 --- a/print_results +++ b/print_results @@ -5,6 +5,9 @@ export TEST_DIR="test_results" echo "Test results" echo "Basic compositor" -./print_subtest "test_results" "" +./psummary "test_results" "" echo "WebRender" -./print_subtest "test_results" "-wr" +./psummary "test_results" "-wr" + +./perrors "test_results" "" +./perrors "test_results" "-wr" diff --git a/print_subtest b/print_subtest deleted file mode 100755 index f64fc8f..0000000 --- a/print_subtest +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/bash -# Analyze and print specialized (basic/webrender) test results - -TEST_DIR=$1 -TEST_FLAVOUR=$2 - -MPASS=`grep "TEST_END: Test OK" $TEST_DIR/mochitest$TEST_FLAVOUR | wc -l` -MERR=`grep "TEST_END: Test ERROR" $TEST_DIR/mochitest$TEST_FLAVOUR | wc -l` -MUNEX=`grep "TEST-UNEXPECTED-FAIL" $TEST_DIR/mochitest$TEST_FLAVOUR | wc -l` -echo "Mochitest PASSED: $MPASS FAILED: $MERR UNEXPECTED-FAILURES: $MUNEX" - -XPCPASS=`grep --text "Expected results:" $TEST_DIR/xpcshell$TEST_FLAVOUR | cut -d ' ' -f 3` -XPCFAIL=`grep --text "Unexpected results:" $TEST_DIR/xpcshell$TEST_FLAVOUR | cut -d ' ' -f 3` -echo "XPCShell: PASSED: $XPCPASS FAILED: $XPCFAIL" - -CRPASS=`grep "REFTEST INFO | Successful:" $TEST_DIR/crashtest$TEST_FLAVOUR | cut -d ' ' -f 5` -CRFAIL=`grep "^REFTEST INFO | Unexpected:" $TEST_DIR/crashtest$TEST_FLAVOUR | cut -d ' ' -f 5` -echo "Crashtest: PASSED: $CRPASS FAILED: $CRFAIL" - -RFPASS=`grep --text "REFTEST INFO | Successful:" $TEST_DIR/reftest$TEST_FLAVOUR | cut -d ' ' -f 5` -RFUN=`grep --text "^REFTEST INFO | Unexpected:" $TEST_DIR/reftest$TEST_FLAVOUR | cut -d ' ' -f 5` -RFKNOWN=`grep --text "REFTEST INFO | Known problems:" $TEST_DIR/reftest$TEST_FLAVOUR | cut -d ' ' -f 6` -echo "Reftest: PASSED: $RFPASS FAILED: $RFUN Known issues: $RFKNOWN" diff --git a/psummary b/psummary new file mode 100755 index 0000000..f64fc8f --- /dev/null +++ b/psummary @@ -0,0 +1,23 @@ +#!/usr/bin/bash +# Analyze and print specialized (basic/webrender) test results + +TEST_DIR=$1 +TEST_FLAVOUR=$2 + +MPASS=`grep "TEST_END: Test OK" $TEST_DIR/mochitest$TEST_FLAVOUR | wc -l` +MERR=`grep "TEST_END: Test ERROR" $TEST_DIR/mochitest$TEST_FLAVOUR | wc -l` +MUNEX=`grep "TEST-UNEXPECTED-FAIL" $TEST_DIR/mochitest$TEST_FLAVOUR | wc -l` +echo "Mochitest PASSED: $MPASS FAILED: $MERR UNEXPECTED-FAILURES: $MUNEX" + +XPCPASS=`grep --text "Expected results:" $TEST_DIR/xpcshell$TEST_FLAVOUR | cut -d ' ' -f 3` +XPCFAIL=`grep --text "Unexpected results:" $TEST_DIR/xpcshell$TEST_FLAVOUR | cut -d ' ' -f 3` +echo "XPCShell: PASSED: $XPCPASS FAILED: $XPCFAIL" + +CRPASS=`grep "REFTEST INFO | Successful:" $TEST_DIR/crashtest$TEST_FLAVOUR | cut -d ' ' -f 5` +CRFAIL=`grep "^REFTEST INFO | Unexpected:" $TEST_DIR/crashtest$TEST_FLAVOUR | cut -d ' ' -f 5` +echo "Crashtest: PASSED: $CRPASS FAILED: $CRFAIL" + +RFPASS=`grep --text "REFTEST INFO | Successful:" $TEST_DIR/reftest$TEST_FLAVOUR | cut -d ' ' -f 5` +RFUN=`grep --text "^REFTEST INFO | Unexpected:" $TEST_DIR/reftest$TEST_FLAVOUR | cut -d ' ' -f 5` +RFKNOWN=`grep --text "REFTEST INFO | Known problems:" $TEST_DIR/reftest$TEST_FLAVOUR | cut -d ' ' -f 6` +echo "Reftest: PASSED: $RFPASS FAILED: $RFUN Known issues: $RFKNOWN" diff --git a/run-tests-wayland b/run-tests-wayland index c3833fa..36d874d 100755 --- a/run-tests-wayland +++ b/run-tests-wayland @@ -36,8 +36,8 @@ if [ $OFFSCREEN_TESTING -ne 0 ] ; then done fi -./mach xpcshell-test 2>&1 | cat - | tee $TEST_DIR/xpcshell -./mach xpcshell-test --enable-webrender 2>&1 | cat - | tee $TEST_DIR/xpcshell-wr +./mach xpcshell-test --sequential 2>&1 | cat - | tee $TEST_DIR/xpcshell +./mach xpcshell-test --sequential --enable-webrender 2>&1 | cat - | tee $TEST_DIR/xpcshell-wr # Basic render testing export TEST_PARAMS="" diff --git a/run-tests-x11 b/run-tests-x11 index 86c3d2e..8ec51a5 100755 --- a/run-tests-x11 +++ b/run-tests-x11 @@ -13,8 +13,8 @@ ln -s /usr/bin/pk12util objdir/dist/bin/pk12util NCPUS="`/usr/bin/getconf _NPROCESSORS_ONLN`" -./mach xpcshell-test 2>&1 | cat - | tee $TEST_DIR/xpcshell -./mach xpcshell-test --enable-webrender 2>&1 | cat - | tee $TEST_DIR/xpcshell-wr +./mach xpcshell-test --sequential 2>&1 | cat - | tee $TEST_DIR/xpcshell +./mach xpcshell-test --sequential --enable-webrender 2>&1 | cat - | tee $TEST_DIR/xpcshell-wr # Basic render testing export TEST_PARAMS="" -- cgit