From dd15f72984cd7f8b8766562b42e2073f4758881e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 18 Jan 2023 15:59:47 +0100 Subject: [PATCH] Work around heisen-failures in CI Use pytest-rerunfailures to rerun flaky test runs Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 24 ++++++++++++------------ setup-mini.cfg | 3 ++- setup.cfg | 1 + 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 20fc9f46525..4c5b04fb1e3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,7 +22,7 @@ jobs: venv/bin/pytest -n 3 -vvs --test-suite=all \ --ignore=tests/cluecode \ --ignore=tests/licensedcode \ - --ignore=tests/scancode/test_cli.py + --ignore=tests/scancode/test_cli.py --reruns 2 cluecode_license_base: | venv/bin/pytest -n 3 -vvs --test-suite=all \ @@ -34,7 +34,7 @@ jobs: --ignore=tests/licensedcode/test_detection_datadriven3.py \ --ignore=tests/licensedcode/test_detection_datadriven4.py \ --ignore=tests/licensedcode/test_additional_license.py \ - tests/licensedcode + tests/licensedcode --reruns 2 license_datadriven1_2: | venv/bin/pytest -n 3 -vvs --test-suite=all \ @@ -77,7 +77,7 @@ jobs: license_cache: | venv/bin/pytest -n 3 -vvs --test-suite=all \ - tests/licensedcode/test_zzzz_cache.py + tests/licensedcode/test_zzzz_cache.py --reruns 2 # this test runs in isolation because it modifies the actual # license index with additional licenses provided by a plugin @@ -98,7 +98,7 @@ jobs: python_versions: ['3.8', '3.9', '3.10', '3.11'] python_architecture: x64 test_suites: - all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2 - template: etc/ci/azure-posix.yml parameters: @@ -107,7 +107,7 @@ jobs: python_versions: ['3.8', '3.9', '3.10', '3.11'] python_architecture: x64 test_suites: - all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2 - template: etc/ci/azure-posix.yml parameters: @@ -116,7 +116,7 @@ jobs: python_versions: ['3.8', '3.9', '3.10', '3.11'] python_architecture: x64 test_suites: - all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2 - template: etc/ci/azure-posix.yml parameters: @@ -125,7 +125,7 @@ jobs: python_versions: ['3.8', '3.9', '3.10', '3.11'] python_architecture: x64 test_suites: - all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2 - template: etc/ci/azure-posix.yml parameters: @@ -134,7 +134,7 @@ jobs: python_versions: ['3.8', '3.9', '3.10', '3.11'] python_architecture: x64 test_suites: - all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2 - template: etc/ci/azure-win.yml parameters: @@ -143,7 +143,7 @@ jobs: python_versions: ['3.8'] python_architecture: x64 test_suites: - all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py + all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py --reruns 2 - template: etc/ci/azure-win.yml parameters: @@ -152,7 +152,7 @@ jobs: python_versions: ['3.9', '3.10', '3.11'] python_architecture: x64 test_suites: - all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py + all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py --reruns 2 - template: etc/ci/azure-win.yml parameters: @@ -161,7 +161,7 @@ jobs: python_versions: ['3.8'] python_architecture: x64 test_suites: - all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py + all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py --reruns 2 - template: etc/ci/azure-win.yml parameters: @@ -170,7 +170,7 @@ jobs: python_versions: ['3.9', '3.10', '3.11'] python_architecture: x64 test_suites: - all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py + all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py --reruns 2 ################################################################################ diff --git a/setup-mini.cfg b/setup-mini.cfg index a4587ca38e3..08309ca2d6a 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -1,5 +1,5 @@ [metadata] -name = scancode-toolkit-mini +name = scancode-toolkit version = 32.0.0rc1 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft @@ -133,6 +133,7 @@ testing = black isort vendorize >= 0.3.0 + pytest-rerunfailures docs = Sphinx >= 3.3.1 diff --git a/setup.cfg b/setup.cfg index 58149eb984d..f11f1b072e6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -133,6 +133,7 @@ testing = black isort vendorize >= 0.3.0 + pytest-rerunfailures docs = Sphinx >= 3.3.1