add/enforce custom destructive mark and enforce serial

* adds a new pytest CLI option `--run-destructive`; tests marked `destructive` will skip when run without this option 
* enforces `serial` marking, skips any test so-marked if `numprocesses` is present and not 0 (we can't see `1` with xdist)
* tox will run both integration and pulp-integration tests with non-serial followed by serial
* CI defaults to adding `--run-destructive` on integration/pulp-integration

.github/workflows/ci.yml

@@ -101,7 +101,7 @@ jobs:
 
       - name: Run pulp integration tests
         run: |
-          tox
+          tox -- --run-destructive
 
 
   integration:
@@ -144,9 +144,9 @@ jobs:
         run: |
           tox --notest
 
-      - name: Run integration tests
+      - name: Run integration tests (including destructive)
         run: |
-          tox
+          tox -- --run-destructive
 
       - name: Upload coverage report
         uses: codecov/codecov-action@v3

pytest.ini

@@ -3,6 +3,7 @@ markers =
     run_command: tests that use the run_command utility which runs a subprocess
     test_all_runtimes: Generate a test for each supported container runtime
     serial: Tests that need to run serially
+    destructive: Tests that may potentially be destructive to the host (skipped by default without `--run-destructive`)
 testpaths = test
 addopts =
     -r a
@@ -17,4 +18,3 @@ addopts =
     --cov-report html
     --cov-report term
     --cov-report xml
-

test/conftest.py

@@ -33,6 +33,13 @@ def do_not_run_commands(request, mocker):
 def data_dir():
     return pathlib.Path(pathlib.Path(__file__).parent).joinpath('data')
 
+def pytest_addoption(parser):
+    parser.addoption(
+        '--run-destructive',
+        action='store_true',
+        default=False,
+        help='Run tests that may be destructive to the host'
+    )
 
 @pytest.fixture
 def exec_env_definition_file(tmp_path):
@@ -86,6 +93,18 @@ def _write_file(content={}):
     return _write_file
 
 
+def pytest_collection_modifyitems(session, config, items):
+    # mark destructive items as skipped if `--run-destructive` was not specified
+    if not config.getoption('--run-destructive'):
+        for destructive_item in (i for i in items if any(i.iter_markers(name='destructive'))):
+            destructive_item.add_marker(pytest.mark.skip(reason='test is potentially destructive to the host (add --run-destructive to allow)'))
+
+    # mark serial items as skipped if it looks like we're running with some obvious kinds of parallelism
+    if getattr(config.option, 'numprocesses', None):
+        for serial_item in (i for i in items if any(i.iter_markers(name='serial'))):
+            serial_item.add_marker(pytest.mark.skip(reason='test requires serial execution (add --numprocesses 0 to allow)'))
+
+
 def pytest_generate_tests(metafunc):
     """If a test uses the custom marker ``test_all_runtimes``, generate marks
     for all supported container runtimes. The requires the test to accept

test/integration/test_build.py

@@ -172,7 +172,9 @@ def test_has_pytz(cli, runtime, data_dir, ee_tag, tmp_path):
 
     assert 'World timezone definitions, modern and historical' in result.stdout
 
-
+# can trash the build cache, must be run independently and with the user's explicit consent
+@pytest.mark.serial
+@pytest.mark.destructive
 @pytest.mark.test_all_runtimes
 def test_build_layer_reuse(cli, runtime, data_dir, ee_tag, tmp_path):
     ee_def = data_dir / 'minimal_fast' / 'execution-environment.yml'
@@ -182,7 +184,7 @@ def test_build_layer_reuse(cli, runtime, data_dir, ee_tag, tmp_path):
         cli(f'{runtime} builder prune --force')
 
     build_cmd = f'ansible-builder build -c {tmp_path} -f {ee_def} -t {ee_tag} --container-runtime {runtime} -v 3 --squash off'
-    cli(build_cmd)
+    cli(build_cmd + ' --no-cache')
     result = cli(build_cmd)
 
     # Get the range of lines that contain the step we want to ensure used the cached layer

test/pulp_integration/test_policies.py

@@ -19,6 +19,7 @@
 
 
 @pytest.mark.serial
+@pytest.mark.destructive
 class TestPolicies:
     """
     All tests within this class must run serially since they all make use of

tox.ini

@@ -27,7 +27,7 @@ commands = pytest {posargs:test/unit}
 # (the system policy.json file).
 description = Run pulp integration tests
 commands =
-    pytest -n 1 -m "serial" {posargs:test/pulp_integration}
+    pytest -n 0 -m "serial" {posargs:test/pulp_integration}
     pytest -m "not serial" {posargs:test/pulp_integration}
 
 [testenv:integration{,-py39,-py310,-py311}]
@@ -36,7 +36,9 @@ description = Run integration tests
 passenv =
     HOME
     KEEP_IMAGES
-commands = pytest {posargs:test/integration}
+commands =
+    pytest -m "not serial" {posargs:test/integration}
+    pytest -n 0 -m "serial" {posargs:test/integration}
 
 [testenv:docs]
 description = Build documentation