Move test environment setup into a fixture

Additionally, this will run the test scripts in a temporary directory.
In tmpdir handling in the config.script snippet was removed. It never
worked properly anyway. The pytest tmp_path fixture however provides
automatic cleanup for files older than three runs.
Another result is, that config.script no longer depends on any specific
location of files.

[noissue]

.gitignore

@@ -1,6 +1,6 @@
 *.egg-info
 __pycache__/
 build/
-tests/scripts/config/pulp/settings.toml
+tests/settings.toml
 /.ci/settings/
 !/.ci/settings/settings.py

Makefile

@@ -9,10 +9,11 @@ lint:
 	mypy
 	@echo "🙊 Code 🙈 LGTM 🙉 !"
 
-tests/scripts/config/pulp/settings.toml:
+tests/settings.toml:
 	cp $@.example $@
+	@echo "In order to configure the tests to talk to your test server, you might need to edit $@ ."
 
-test: | tests/scripts/config/pulp/settings.toml
+test: | tests/settings.toml
 	pytest -v tests
 
 .PHONY: black lint

tests/conftest.py

@@ -0,0 +1,42 @@
+import os
+
+import pytest
+import toml
+
+# Constants used in tests
+PULP_FIXTURES_URL = os.environ.get("PULP_FIXTURES_URL", "https://fixtures.pulpproject.org")
+
+ENV_CONSTANTS = {
+    "PULP_FIXTURES_URL": PULP_FIXTURES_URL,
+    "FILE_REMOTE_URL": PULP_FIXTURES_URL + "/file/PULP_MANIFEST",
+    "CONTAINER_REMOTE_URL": "https://registry-1.docker.io",
+    "CONTAINER_IMAGE": "hello-world",
+    "RPM_REMOTE_URL": PULP_FIXTURES_URL + "/rpm-unsigned",
+    "ANSIBLE_COLLECTION_REMOTE_URL": "https://galaxy.ansible.com/",
+    "ANSIBLE_ROLE_REMOTE_URL": "https://galaxy.ansible.com/api/v1/roles/?namespace__name=elastic",
+}
+
+
+@pytest.fixture
+def cli_env(tmp_path, monkeypatch):
+    """
+    This fixture will set up the environment for cli commands by:
+
+    * creating a tmp_dir
+    * placing the config there
+    * pointing XDG_CONFIG_HOME accordingly
+    * supplying other useful environment vars
+    """
+    settings = toml.load("tests/settings.toml")
+    settings_path = tmp_path / "config" / "pulp"
+    settings_path.mkdir(parents=True)
+    with open(settings_path / "settings.toml", "w") as settings_file:
+        toml.dump(settings, settings_file)
+    monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config"))
+    monkeypatch.setenv("PULP_BASE_URL", settings["cli"]["base_url"])
+    monkeypatch.setenv("VERIFY_SSL", str(settings["cli"].get("verify_ssl", True)).lower())
+
+    for key, value in ENV_CONSTANTS.items():
+        monkeypatch.setenv(key, value)
+
+    yield settings

tests/scripts/config.source

@@ -1,26 +1,5 @@
-# Provide values from config
-settings="$(dirname "$(realpath "$0")")/config/pulp/settings.toml"
-PULP_BASE_URL="$(sed -n -e 's/^base_url\s*=\s*"\(\S*\)"\s*$/\1/p' "$settings")"
-VERIFY_SSL="$(sed -n -e 's/^verify_ssl\s*=\s*\(\S*\)\s*$/\1/p' "$settings")"
-
-# Configure path to config
-export XDG_CONFIG_HOME="$(dirname "$(realpath "$0")")/config"
-
-# Constants used in tests
-PULPCORE_VERSION=$(pulp status | jq -r '.versions | .[] | select(.component=="pulpcore").version')
-PULP_FIXTURES_URL="${PULP_FIXTURES_URL:-https://fixtures.pulpproject.org}"
-FILE_REMOTE_URL="${PULP_FIXTURES_URL}/file/PULP_MANIFEST"
-CONTAINER_REMOTE_URL="https://registry-1.docker.io"
-CONTAINER_IMAGE="hello-world"
-RPM_REMOTE_URL="${PULP_FIXTURES_URL}/rpm-unsigned"
-ANSIBLE_COLLECTION_REMOTE_URL="https://galaxy.ansible.com/"
-ANSIBLE_ROLE_REMOTE_URL="https://galaxy.ansible.com/api/v1/roles/?namespace__name=elastic"
-
 # Library for test helper functions
 
-TMP="$(mktemp -d)"
-trap "rm -rf $TMP" EXIT
-
 # open fd 3 as a copy of stderr
 exec 3<&2
 
@@ -30,17 +9,17 @@ exec 3<&2
 expect_succ () {
   if {
     "$@"
-  } 1>"${TMP}/log.out" 2>"${TMP}/log.err"
+  } 1>log.out 2>log.err
   then
     echo "SUCCESS [$@]" >&3
-    OUTPUT="$(cat "${TMP}/log.out")"
-    ERROUTPUT="$(cat "${TMP}/log.err")"
+    OUTPUT="$(cat log.out)"
+    ERROUTPUT="$(cat log.err)"
   else
     echo "FAILURE [$@]" >&3
     echo "=== STDOUT ===" >&3
-    cat "${TMP}/log.out" >&3
+    cat log.out >&3
     echo "=== STDERR ===" >&3
-    cat "${TMP}/log.err" >&3
+    cat log.err >&3
     echo "==============" >&3
     false
   fi
@@ -52,18 +31,18 @@ expect_succ () {
 expect_fail () {
   if {
     "$@"
-  } 1>"${TMP}/log.out" 2>"${TMP}/log.err"
+  } 1>log.out 2>log.err
   then
     echo "FAILURE [! $@]" >&3
     echo "=== STDOUT ===" >&3
-    cat "${TMP}/log.out" >&3
+    cat log.out >&3
     echo "=== STDERR ===" >&3
-    cat "${TMP}/log.err" >&3
+    cat log.err >&3
     false
   else
     echo "SUCCESS [! $@]" >&3
-    OUTPUT="$(cat "${TMP}/log.out")"
-    ERROUTPUT="$(cat "${TMP}/log.err")"
+    OUTPUT="$(cat log.out)"
+    ERROUTPUT="$(cat log.err)"
   fi
 }
 

tests/scripts/config/pulp/settings.toml

@@ -0,0 +1,7 @@
+[cli]
+# base_url = "http://localhost:8080"
+base_url = "https://pulp3-sandbox-debian-testing"
+# base_url = "http://pulp3-source-debian-testing"
+username = "admin"
+password = "password"
+verify_ssl = false

tests/scripts/test_artifact.sh

@@ -4,7 +4,6 @@
 . "$(dirname "$(realpath "$0")")/config.source"
 
 cleanup() {
-  rm test.txt
   pulp orphans delete
 }
 trap cleanup EXIT

tests/scripts/test_config.sh

@@ -4,19 +4,15 @@
 . "$(dirname "$(realpath "$0")")/config.source"
 
 good_settings="${XDG_CONFIG_HOME}/pulp/settings.toml"
-bad_settings="${XDG_CONFIG_HOME}/pulp/bad_settings.toml"
+bad_settings="bad_settings.toml"
 test_settings="test.toml"
 
 export XDG_CONFIG_HOME=/nowhere
 
-cleanup() {
-  rm $test_settings
-}
-trap cleanup EXIT
-
-
 # SETTINGS OPTION
 
+sed -e '/base_url/c\base_url = "http://badurl"' "$good_settings" > "$bad_settings"
+
 expect_succ pulp --config "$good_settings" file repository list
 expect_succ pulp --config "$bad_settings" --base-url "$PULP_BASE_URL" file repository list
 

tests/scripts/test_file_content.sh

@@ -6,7 +6,6 @@
 pulp debug has-plugin --name "pulp_file" || exit 3
 
 cleanup() {
-  rm test.txt
   pulp file repository destroy --name "cli_test_file_repository" || true
   pulp orphans delete || true
 }

tests/scripts/test_file_content_bulk.sh

@@ -6,8 +6,6 @@
 pulp debug has-plugin --name "pulp_file" || exit 3
 
 cleanup() {
-  rm test_1.txt test_2.txt test_3.txt
-  rm add_content.json remove_content.json
   pulp file repository destroy --name "cli_test_file_repository" || true
   pulp orphans delete || true
 }

tests/test_scripts.py

@@ -11,8 +11,11 @@
 
 
 @pytest.mark.parametrize("test_name", TEST_NAMES)
-def test_script(test_name):
-    run = subprocess.run([os.path.join("tests", "scripts", "test_" + test_name + ".sh")])
+def test_script(test_name, cli_env, tmp_path):
+    run = subprocess.run(
+        [os.path.realpath(os.path.join("tests", "scripts", "test_" + test_name + ".sh"))],
+        cwd=tmp_path,
+    )
     if run.returncode == 3:
         pytest.skip("Skipped as requested by the script.")
     assert run.returncode == 0