diff --git a/testing/mochitest/mochitest_options.py b/testing/mochitest/mochitest_options.py index eae6c7b81792..8524b2a08bd4 100644 --- a/testing/mochitest/mochitest_options.py +++ b/testing/mochitest/mochitest_options.py @@ -6078,6 +6078,60 @@ } +] + +[ + +[ +" +- +- +restart +- +after +- +failure +" +] + +{ + +" +dest +" +: +" +restartAfterFailure +" + +" +default +" +: +False + +" +help +" +: +" +Terminate +the +session +on +first +failure +and +restart +where +you +left +off +. +" + +} + ] ] diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index ebd66a3ab287..75543383c12a 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -18536,6 +18536,10 @@ = None +restartAfterFailure += +False + marionette_args = None @@ -19045,6 +19049,10 @@ bisectChunk = bisectChunk + +restartAfterFailure += +restartAfterFailure ) @@ -20838,6 +20846,109 @@ result ) +elif +options +. +restartAfterFailure +: + +if +not +self +. +expectedError +: + +status += +- +1 + +else +: + +firstFail += +len +( +testsToRun +) + +for +key +in +self +. +expectedError +: + +full_key += +[ +x +for +x +in +testsToRun +if +key +in +x +] + +if +full_key +: + +if +testsToRun +. +index +( +full_key +[ +0 +] +) +< +firstFail +: + +firstFail += +testsToRun +. +index +( +full_key +[ +0 +] +) + +testsToRun += +testsToRun +[ +firstFail ++ +1 +: +] + +if +testsToRun += += +[ +] +: + +status += +- +1 + else : @@ -24756,6 +24867,12 @@ . bisectChunk +restartAfterFailure += +options +. +restartAfterFailure + marionette_args = marionette_args @@ -25808,6 +25925,10 @@ bisectChunk = None + +restartAfterFailure += +None ) : @@ -25885,6 +26006,12 @@ = bisectChunk +self +. +restartAfterFailure += +restartAfterFailure + self . browserProcessId @@ -26036,6 +26163,10 @@ self . bisectChunk +or +self +. +restartAfterFailure : handlers @@ -27444,6 +27575,18 @@ = True +if +options +. +restartAfterFailure +: + +options +. +runUntilFailure += +True + if options . diff --git a/testing/mochitest/runtestsremote.py b/testing/mochitest/runtestsremote.py index 46689a5c6160..c1edc161c380 100644 --- a/testing/mochitest/runtestsremote.py +++ b/testing/mochitest/runtestsremote.py @@ -2201,6 +2201,10 @@ = None +restartAfterFailure += +False + marionette_args = None diff --git a/testing/mochitest/tests/SimpleTest/TestRunner.js b/testing/mochitest/tests/SimpleTest/TestRunner.js index 10a0c9f68d91..279dbc7000e9 100644 --- a/testing/mochitest/tests/SimpleTest/TestRunner.js +++ b/testing/mochitest/tests/SimpleTest/TestRunner.js @@ -980,6 +980,11 @@ _numTimeouts TestRunner . maxTimeouts +| +| +TestRunner +. +runUntilFailure ) { TestRunner diff --git a/testing/mochitest/tests/python/conftest.py b/testing/mochitest/tests/python/conftest.py index bf6eca52290d..e6e0e37b6344 100644 --- a/testing/mochitest/tests/python/conftest.py +++ b/testing/mochitest/tests/python/conftest.py @@ -335,6 +335,31 @@ " ) +restartAfterFailure += +False + +if +" +restartAfterFailure +" +in +request +. +fixturenames +: + +restartAfterFailure += +request +. +getfixturevalue +( +" +restartAfterFailure +" +) + setup_test_harness ( * @@ -497,6 +522,12 @@ : runFailures +" +restartAfterFailure +" +: +restartAfterFailure + " keep_open " @@ -678,21 +709,37 @@ ) : +if +isinstance +( +test +str +) +: + +test += +[ +test +] + return -{ +[ +{ + " name " : -test - +t + " relpath " : -test - +t + " path " @@ -704,9 +751,9 @@ join ( test_root -test +t ) - + " manifest " @@ -720,13 +767,13 @@ test_root manifest_name ) - + " manifest_relpath " : manifest_name - + " skip - @@ -734,8 +781,15 @@ " : runFailures - + } + +for +t +in +test + +] def inner @@ -799,6 +853,9 @@ tests ) ) +[ +0 +] ) options diff --git a/testing/mochitest/tests/python/files/browser_fail2.js b/testing/mochitest/tests/python/files/browser_fail2.js new file mode 100644 index 000000000000..47a2454bce4a --- /dev/null +++ b/testing/mochitest/tests/python/files/browser_fail2.js @@ -0,0 +1,16 @@ +function +test +( +) +{ +ok +( +false +" +Test +is +ok +" +) +; +} diff --git a/testing/mochitest/tests/python/files/test_fail2.html b/testing/mochitest/tests/python/files/test_fail2.html new file mode 100644 index 000000000000..6e5448178178 --- /dev/null +++ b/testing/mochitest/tests/python/files/test_fail2.html @@ -0,0 +1,219 @@ +< +! +DOCTYPE +HTML +> +< +html +> +< +! +- +- +https +: +/ +/ +bugzilla +. +mozilla +. +org +/ +show_bug +. +cgi +? +id += +1343659 +- +- +> +< +head +> +< +meta +charset += +" +utf +- +8 +" +> +< +title +> +Test +Fail +< +/ +title +> +< +script +src += +" +/ +tests +/ +SimpleTest +/ +SimpleTest +. +js +" +> +< +/ +script +> +< +link +rel += +" +stylesheet +" +type += +" +text +/ +css +" +href += +" +/ +tests +/ +SimpleTest +/ +test +. +css +" +/ +> +< +script +type += +" +application +/ +javascript +" +> +ok +( +false +" +Test +is +ok +" +) +; +< +/ +script +> +< +/ +head +> +< +body +> +< +a +target += +" +_blank +" +href += +" +https +: +/ +/ +bugzilla +. +mozilla +. +org +/ +show_bug +. +cgi +? +id += +1343659 +" +> +Mozilla +Bug +1343659 +< +/ +a +> +< +p +id += +" +display +" +> +< +/ +p +> +< +div +id += +" +content +" +style += +" +display +: +none +" +> +< +/ +div +> +< +pre +id += +" +test +" +> +< +/ +pre +> +< +/ +body +> +< +/ +html +> diff --git a/testing/mochitest/tests/python/test_mochitest_integration.py b/testing/mochitest/tests/python/test_mochitest_integration.py index 8f3cc9c599d7..b2943c56e4ed 100644 --- a/testing/mochitest/tests/python/test_mochitest_integration.py +++ b/testing/mochitest/tests/python/test_mochitest_integration.py @@ -931,6 +931,282 @@ . mark . +parametrize +( +" +runFailures +" +[ +" +selftest +" +" +" +] +) +pytest +. +mark +. +parametrize +( +" +flavor +" +[ +" +plain +" +" +browser +- +chrome +" +] +) +def +test_output_restart_after_failure +( +flavor +runFailures +runtests +test_name +) +: + +extra_opts += +{ +} + +results += +{ + +" +status +" +: +0 +if +runFailures +else +1 + +" +tbpl_status +" +: +TBPL_SUCCESS +if +runFailures +else +TBPL_WARNING + +" +log_level +" +: +( +INFO +WARNING +) + +" +lines +" +: +2 + +" +line_status +" +: +" +PASS +" +if +runFailures +else +" +FAIL +" + +} + +extra_opts +[ +" +restartAfterFailure +" +] += +True + +if +runFailures +: + +extra_opts +[ +" +runFailures +" +] += +runFailures + +extra_opts +[ +" +crashAsPass +" +] += +True + +extra_opts +[ +" +timeoutAsPass +" +] += +True + +tests += +[ +test_name +( +" +fail +" +) +test_name +( +" +fail2 +" +) +] + +status +lines += +runtests +( +tests +* +* +extra_opts +) + +assert +status += += +results +[ +" +status +" +] + +tbpl_status +log_level +summary += +get_mozharness_status +( +lines +status +) + +assert +tbpl_status += += +results +[ +" +tbpl_status +" +] + +assert +log_level +in +results +[ +" +log_level +" +] + +start_lines += +[ + +line +for +line +in +lines +if +" +Application +command +: +" +in +line +. +get +( +" +message +" +" +" +) + +] + +if +not +runFailures +: + +assert +len +( +start_lines +) += += +results +[ +" +lines +" +] + +else +: + +assert +len +( +start_lines +) += += +1 +pytest +. +mark +. skip_mozinfo ( "