Group tests in folders

[noissue]

README.md

@@ -91,9 +91,19 @@ As a rule of thumb, all necessary workarounds should be implemented in the corre
 Tests are shell scripts in `tests/scripts` with names like `test_*.sh`.
 They should should focus on the cli operation and are not a replacement for pulp integration tests;
 i.e. make sure the cli translates to the right api calls, but do not care about pulp internals.
-In order to perform, a running instance of pulp with all necessary plugins installed must be configured in `tests/scripts/config/pulp/settings.toml`.
-All tests are excercised using `make test`.
-Individual tests can be selected by pattern via `pytest -k file`.
+
+#### Running Tests
+
+In order to run tests, a running instance of pulp with all necessary plugins installed must be
+configured in `tests/settings.toml`.
+
+To run tests:
+
+```
+make test                           # all tests
+pytest -m pulp_file                 # tests for pulp_file
+pytest -m pulp_file -k test_remote  # run tests/scripts/pulp_file/test_remote.sh
+```
 
 ## Releasing
 

pyproject.toml

@@ -12,3 +12,12 @@ line-length = 100
 [tool.isort]
 profile = "black"
 line_length = 100
+
+[tool.pytest.ini_options]
+markers = [
+  "script: Marks tests provided as shell scripts",
+  "pulpcore: pulpcore tests.",
+  "pulp_file: pulp_file tests.",
+  "pulp_ansible: pulp_ansible tests.",
+  "pulp_container: pulp_container tests."
+]

tests/scripts/test_ansible_distribution.sh → tests/scripts/pulp_ansible/test_distribution.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_ansible" || exit 3
 

tests/scripts/test_ansible_remote.sh → tests/scripts/pulp_ansible/test_remote.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_ansible" || exit 3
 

tests/scripts/test_ansible_repository.sh → tests/scripts/pulp_ansible/test_repository.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_ansible" || exit 3
 

tests/scripts/test_ansible_sync.sh → tests/scripts/pulp_ansible/test_sync.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_ansible" || exit 3
 

tests/scripts/test_container_distribution.sh → tests/scripts/pulp_container/test_distribution.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_container" || exit 3
 

tests/scripts/test_container_remote.sh → tests/scripts/pulp_container/test_remote.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_container" || exit 3
 

tests/scripts/test_container_repository.sh → tests/scripts/pulp_container/test_repository.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_container" || exit 3
 

tests/scripts/test_container_sync.sh → tests/scripts/pulp_container/test_sync.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_container" || exit 3
 

tests/scripts/test_file_content.sh → tests/scripts/pulp_file/test_content.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/scripts/test_file_content_bulk.sh → tests/scripts/pulp_file/test_content_bulk.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/scripts/test_file_distribution.sh → tests/scripts/pulp_file/test_distribution.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/scripts/test_label.sh → tests/scripts/pulp_file/test_label.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulpcore" --min-version "3.10.0" || exit 3
 

tests/scripts/test_file_publication.sh → tests/scripts/pulp_file/test_publication.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/scripts/test_file_remote.sh → tests/scripts/pulp_file/test_remote.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/scripts/test_file_repository.sh → tests/scripts/pulp_file/test_repository.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/scripts/test_file_sync.sh → tests/scripts/pulp_file/test_sync.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/scripts/test_artifact.sh → tests/scripts/pulpcore/test_artifact.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 cleanup() {
   pulp orphans delete

tests/scripts/test_config.sh → tests/scripts/pulpcore/test_config.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 good_settings="${XDG_CONFIG_HOME}/pulp/settings.toml"
 bad_settings="bad_settings.toml"

tests/scripts/test_debug_api.sh → tests/scripts/pulpcore/test_debug_api.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 expect_succ pulp -v status
 

tests/scripts/test_group.sh → tests/scripts/pulpcore/test_group.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 cleanup() {
   pulp group destroy --name "cli_test_group" || true

tests/scripts/test_orphans.sh → tests/scripts/pulpcore/test_orphans.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 expect_succ pulp orphans delete
 

tests/scripts/test_output.sh → tests/scripts/pulpcore/test_output.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 # skip this test if pygments is not installed
 pip show pygments > /dev/null || exit 3

tests/scripts/test_pulpexporter.sh → tests/scripts/pulpcore/test_pulpexporter.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/scripts/test_pulpimporter.sh → tests/scripts/pulpcore/test_pulpimporter.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/scripts/test_rpm_sync_publish.sh → tests/scripts/pulpcore/test_rpm_sync_publish.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_rpm" || exit 3
 

tests/scripts/test_show.sh → tests/scripts/pulpcore/test_show.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/scripts/test_status.sh → tests/scripts/pulpcore/test_status.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 expect_succ pulp status
 

tests/scripts/test_task.sh → tests/scripts/pulpcore/test_task.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # shellcheck source=tests/scripts/config.source
-. "$(dirname "$(realpath "$0")")/config.source"
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
 pulp debug has-plugin --name "pulp_file" || exit 3
 

tests/test_scripts.py

@@ -1,21 +1,22 @@
-import os
 import subprocess
+from pathlib import Path
 
 import pytest
 
-TEST_NAMES = [
-    name[5:-3]
-    for name in os.listdir("tests/scripts")
-    if name.startswith("test_") and name.endswith(".sh")
+TEST_SCRIPTS = [
+    pytest.param(
+        path.resolve(),
+        id=path.stem,
+        marks=[] if path.parent.name == "scripts" else getattr(pytest.mark, path.parent.name),
+    )
+    for path in Path("tests/scripts").glob("**/test_*.sh")
 ]
 
 
-@pytest.mark.parametrize("test_name", TEST_NAMES)
-def test_script(test_name, cli_env, tmp_path):
-    run = subprocess.run(
-        [os.path.realpath(os.path.join("tests", "scripts", "test_" + test_name + ".sh"))],
-        cwd=tmp_path,
-    )
+@pytest.mark.script
+@pytest.mark.parametrize("script", TEST_SCRIPTS)
+def test_script(script, cli_env, tmp_path):
+    run = subprocess.run([script], cwd=tmp_path)
     if run.returncode == 3:
         pytest.skip("Skipped as requested by the script.")
     assert run.returncode == 0