Fix path overrides in python3 when generating the testconfig.

With all the many lines of development, branches and recent shuffling
somewhere this went missing - handle fallout from config generation
changes that make auto path arguments always be from the base root
instead of relative the output directory. Opt to patch the testconfig
generator rather than fiddle with generateconfs again.

fixup

fixup

fixup

fixup

!!! seems isolated directories work locally as of this commit !!!

force everything into the tests output directory

explain allowig absolute paths in temppath()

fixup

fixup

ci++

fixup

fixup

fixup

invoke pip via the active local python binary

fixup

fixup

fixup

fixup

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
           make dependencies
       - name: Run tests
         run: |
-          make test
+          make unittest
 
   python3-rocky9ish:
     runs-on: ubuntu-22.04
@@ -51,7 +51,7 @@ jobs:
           make dependencies
       - name: Run tests
         run: |
-          make test
+          make unittest
 
   python3-rocky8ish:
     runs-on: ubuntu-20.04
@@ -67,7 +67,7 @@ jobs:
           make dependencies
       - name: Run tests
         run: |
-          make test
+          make unittest
 
   python2-latest:
     runs-on: ubuntu-latest
@@ -80,8 +80,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Setup environment
         run: |
-          pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
+          make PYTHON_BIN=python PY=2 dependencies
       - name: Run tests
         run: |
-          PYTHON_BIN=python ./envhelp/makeconfig test --python2
-          MIG_ENV='local' python -m unittest discover -s tests/
+          make PYTHON_BIN=python PY=2 unittest

Makefile

@@ -1,11 +1,22 @@
 ifndef MIG_ENV
 	MIG_ENV = 'local'
 endif
-ifeq ($(PY),2)
+
+ifndef PY
+	PY = 3
+endif
+
+LOCAL_PYTHON_BIN = './envhelp/lpython'
+LOCAL_DEPENDS = './envhelp/local.depends'
+
+ifdef PYTHON_BIN
+	LOCAL_PYTHON_BIN = $(PYTHON_BIN)
+else ifeq ($(PY),2)
 	PYTHON_BIN = './envhelp/python2'
 else
 	PYTHON_BIN = './envhelp/python3'
 endif
+
 ifeq ($(ALLDEPS),1)
 	REQS_PATH = ./recommended.txt
 else
@@ -26,12 +37,13 @@ ifneq ($(MIG_ENV),'local')
 	@echo "unavailable outside local development environment"
 	@exit 1
 endif
-	$(PYTHON_BIN) -m autopep8 --ignore E402 -i
+	$(LOCAL_PYTHON_BIN) -m autopep8 --ignore E402 -i
 
 .PHONY: clean
 clean:
 	@rm -f ./envhelp/py2.imageid
-	@rm -f ./envhelp/py3.depends
+	@rm -f ./envhelp/py3.imageid
+	@rm -f $(LOCAL_DEPENDS)
 
 .PHONY: distclean
 distclean: clean
@@ -44,37 +56,41 @@ distclean: clean
 test: dependencies testconfig
 	@$(PYTHON_BIN) -m unittest discover -s tests/
 
+.PHONY: unittest
+unittest: dependencies testconfig
+	@$(LOCAL_PYTHON_BIN) -m unittest discover -s tests/
+
 .PHONY: dependencies
-dependencies: ./envhelp/venv/pyvenv.cfg ./envhelp/py3.depends
+ifeq ($(PY),2)
+dependencies: ./envhelp/local.depends
+else
+dependencies: ./envhelp/venv/pyvenv.cfg ./envhelp/local.depends
+endif
 
 .PHONY: testconfig
 testconfig: ./envhelp/output/testconfs
 
 ./envhelp/output/testconfs:
-	@./envhelp/makeconfig test --python2
+	@./envhelp/makeconfig test --docker
 	@./envhelp/makeconfig test
-	@mkdir -p ./envhelp/output/certs
-	@mkdir -p ./envhelp/output/state
-	@mkdir -p ./envhelp/output/state/log
 
 ifeq ($(MIG_ENV),'local')
-./envhelp/py3.depends: $(REQS_PATH) local-requirements.txt
+./envhelp/local.depends: $(REQS_PATH) local-requirements.txt
 else
-./envhelp/py3.depends: $(REQS_PATH)
+./envhelp/local.depends: $(REQS_PATH)
 endif
-	@rm -f ./envhelp/py3.depends
-	@echo "upgrading venv pip as required for some dependencies"
-	@./envhelp/venv/bin/pip3 install --upgrade pip
 	@echo "installing dependencies from $(REQS_PATH)"
-	@./envhelp/venv/bin/pip3 install -r $(REQS_PATH)
+	@$(LOCAL_PYTHON_BIN) -m pip install -r $(REQS_PATH)
 ifeq ($(MIG_ENV),'local')
 	@echo ""
 	@echo "installing development dependencies"
-	@./envhelp/venv/bin/pip3 install -r local-requirements.txt
+	@$(LOCAL_PYTHON_BIN) -m pip install -r local-requirements.txt
 endif
-	@touch ./envhelp/py3.depends
+	@touch $(LOCAL_DEPENDS)
 
 ./envhelp/venv/pyvenv.cfg:
 	@echo "provisioning environment"
 	@/usr/bin/env python3 -m venv ./envhelp/venv
-	@rm -f ./envhelp/py3.depends
+	@rm -f $(LOCAL_DEPENDS)
+	@echo "upgrading venv pip as required for some dependencies"
+	@./envhelp/venv/bin/pip3 install --upgrade pip

envhelp/docker/Dockerfile.py3

@@ -0,0 +1,8 @@
+FROM python:3.9
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt local-requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
+
+CMD [ "python", "--version" ]

envhelp/dpython

@@ -0,0 +1,84 @@
+#!/bin/sh
+#
+# --- BEGIN_HEADER ---
+#
+# dpython - wrapper to invoke a containerised python
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+set -e
+
+SCRIPT_PATH=$(realpath "$0")
+SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
+MIG_BASE=$(realpath "$SCRIPT_BASE/..")
+
+if [ -z "${PY}" ]; then
+    echo "No python version specified - please supply a PY env var"
+    exit 1
+fi
+
+PYTHON_SUFFIX="py$PY"
+DOCKER_IMAGEID_FILE="$SCRIPT_BASE/$PYTHON_SUFFIX.imageid"
+DOCKER_PYTHON_BIN="python$PY"
+
+# NOTE: portable dynamic lookup with docker as default and fallback to podman
+DOCKER_BIN=$(command -v docker || command -v podman || echo "")
+DOCKER_BASE="$SCRIPT_BASE/docker"
+DOCKER_FILE="$DOCKER_BASE/Dockerfile.$PYTHON_SUFFIX"
+if [ -z "${DOCKER_BIN}" ]; then
+    echo "No docker binary found - cannot use for python $PY tests"
+    exit 1
+fi
+
+# default PYTHONPATH such that directly executing files in the repo "just works"
+# NOTE: this is hard-coded to the mount point used within the container
+PYTHONPATH='/usr/src/app'
+
+# default any variables for container development
+MIG_ENV=${MIG_ENV:-'docker'}
+
+# determine if the image has changed
+echo -n "validating container.. "
+
+# load a previously written docker image id if present
+IMAGEID_STORED=$(cat "$DOCKER_IMAGEID_FILE" 2>/dev/null || echo "")
+
+IMAGEID=$(${DOCKER_BIN} build -f "$DOCKER_FILE" . -q)
+if [ "$IMAGEID" != "$IMAGEID_STORED" ]; then
+    echo "rebuilt for changes"
+
+    # reset the image id so the next call finds no changes
+    echo "$IMAGEID" > "$DOCKER_IMAGEID_FILE"
+else
+    echo "no changes needed"
+fi
+
+echo "running with MIG_ENV='$MIG_ENV' under python $PY"
+echo
+
+# execute python2 within the image passing the supplied arguments
+
+${DOCKER_BIN} run -it --rm \
+    --mount "type=bind,source=$MIG_BASE,target=/usr/src/app" \
+    --env "PYTHONPATH=$PYTHONPATH" \
+    --env "MIG_ENV=$MIG_ENV" \
+    "$IMAGEID" $DOCKER_PYTHON_BIN $@

envhelp/lpython

@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# --- BEGIN_HEADER ---
+#
+# python3 - wrapper to invoke a local python3 virtual environment
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+set -e
+
+SCRIPT_PATH=$(realpath "$0")
+SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
+MIG_BASE=$(realpath "$SCRIPT_BASE/..")
+
+PYTHON_BIN=${PYTHON_BIN:-"$SCRIPT_BASE/venv/bin/python3"}
+if [ ! -f "${PYTHON_BIN}" ]; then
+    echo "No python binary found - perhaps the virtual env was not created"
+    exit 1
+fi
+
+# default PYTHONPATH such that directly executing files in the repo "just works"
+PYTHONPATH=${PYTHONPATH:-"$MIG_BASE"}
+
+# default any variables for local development
+MIG_ENV=${MIG_ENV:-'local'}
+
+echo "running with MIG_ENV='$MIG_ENV' under local python 3"
+echo
+
+PYTHONPATH="$PYTHONPATH" MIG_ENV="$MIG_ENV" "$PYTHON_BIN" "$@"

envhelp/makeconfig.py

@@ -38,8 +38,9 @@
 
 from mig.shared.install import MIG_BASE, generate_confs
 
-_LOCAL_ENVHELP_OUTPUT_DIR = os.path.realpath(
-    os.path.join(os.path.dirname(__file__), "output"))
+_LOCAL_MIG_BASE = os.path.normpath(
+    os.path.join(os.path.dirname(__file__), ".."))
+_LOCAL_ENVHELP_OUTPUT_DIR = os.path.join(_LOCAL_MIG_BASE, "envhelp/output")
 _MAKECONFIG_ALLOWED = ["local", "test"]
 
 
@@ -51,21 +52,27 @@ def _at(sequence, index=-1, default=None):
         return default
 
 
-def write_testconfig(env_name, is_py2=False):
-    confs_name = 'confs' if env_name == 'local' else '%sconfs' % (env_name,)
-    confs_suffix = 'py2' if is_py2 else 'py3'
+def write_testconfig(env_name, is_docker=False):
+    is_predefined = env_name == 'test'
+    confs_name = '%sconfs' % (env_name,)
+    if is_predefined:
+        confs_suffix = 'docker' if is_docker else 'local'
+    else:
+        confs_suffix = 'py3'
 
     overrides = {
         'destination': os.path.join(_LOCAL_ENVHELP_OUTPUT_DIR, confs_name),
         'destination_suffix': "-%s" % (confs_suffix,),
     }
 
-    # determine the paths by which we will access the various configured dirs
-    if is_py2:
+    # determine the paths b which we will access the various configured dirs
+    #  the tests output directory - when invoked within
+
+    if is_predefined and is_docker:
         env_mig_base = '/usr/src/app'
     else:
-        env_mig_base = MIG_BASE
-    conf_dir_path = os.path.join(env_mig_base, "envhelp/output")
+        env_mig_base = _LOCAL_MIG_BASE
+    conf_dir_path = os.path.join(env_mig_base, "tests/output")
 
     overrides.update(**{
         'mig_code': os.path.join(conf_dir_path, 'mig'),
@@ -85,7 +92,7 @@ def write_testconfig(env_name, is_py2=False):
 
 def main_(argv):
     env_name = _at(argv, index=1, default='')
-    arg_is_py2 = '--python2' in argv
+    arg_is_docker = '--docker' in argv
 
     if env_name == '':
         raise RuntimeError(
@@ -94,7 +101,7 @@ def main_(argv):
         raise RuntimeError('environment must be one of %s' %
                            (_MAKECONFIG_ALLOWED,))
 
-    write_testconfig(env_name, is_py2=arg_is_py2)
+    write_testconfig(env_name, is_docker=arg_is_docker)
 
 
 def main(argv=sys.argv):

envhelp/python2

@@ -29,46 +29,5 @@ set -e
 
 SCRIPT_PATH=$(realpath "$0")
 SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
-DOCKER_BASE="$SCRIPT_BASE/docker"
-DOCKER_IMAGEID_FILE="$SCRIPT_BASE/py2.imageid"
-# NOTE: portable dynamic lookup with docker as default and fallback to podman
-DOCKER_BIN=$(command -v docker || command -v podman || echo "")
 
-if [ -z "${DOCKER_BIN}" ]; then
-    echo "No docker binary found - cannot use for python2 tests"
-    exit 1
-fi
-
-# default PYTHONPATH such that directly executing files in the repo "just works"
-# NOTE: this is hard-coded to the mount point used within the container
-PYTHONPATH='/usr/app/src'
-
-# default any variables for local development
-MIG_ENV=${MIG_ENV:-'local'}
-
-# determine if the image has changed
-echo -n "validating container.. "
-
-# load a previously written docker image id if present
-IMAGEID_STORED=$(cat "$DOCKER_IMAGEID_FILE" 2>/dev/null || echo "")
-
-IMAGEID=$(${DOCKER_BIN} build -f "$DOCKER_BASE/Dockerfile.python2" . -q)
-if [ "$IMAGEID" != "$IMAGEID_STORED" ]; then
-    echo "rebuilt for changes"
-
-    # reset the image id so the next call finds no changes
-    echo "$IMAGEID" > "$DOCKER_IMAGEID_FILE"
-else
-    echo "no changes needed"
-fi
-
-echo "running with MIG_ENV='$MIG_ENV' under python 2"
-echo
-
-# execute python2 within the image passing the supplied arguments
-
-${DOCKER_BIN} run -it --rm \
-    --mount type=bind,source=.,target=/usr/src/app \
-    --env "PYTHONPATH=$PYTHON_PATH" \
-    --env "MIG_ENV=$MIG_ENV" \
-    "$IMAGEID" python2 "$@"
+PY=2 $SCRIPT_BASE/dpython "$@"