Updated helix.spark image to stable release 3.5.1.3

Makefile

@@ -12,6 +12,10 @@ devdocker: ## Builds the docker for dev
 .PHONY:init
 init: devdocker up setup-pre-commit  ## Initializes the local developer environment
 
+.PHONY:build
+build: ## Builds the docker for dev
+	docker compose build --progress=plain --parallel
+
 .PHONY: up
 up: Pipfile.lock
 	docker compose up --build -d
@@ -35,9 +39,10 @@ run-pre-commit: setup-pre-commit
 	./.git/hooks/pre-commit
 
 .PHONY:update
-update: down Pipfile.lock setup-pre-commit  ## Updates all the packages using Pipfile
+update: Pipfile.lock setup-pre-commit  ## Updates all the packages using Pipfile
 	docker compose run --rm --name sfs_pipenv dev pipenv sync --dev && \
-	make devdocker
+	make devdocker && \
+	make pipenv-setup
 
 .PHONY:tests
 tests: up
@@ -51,6 +56,11 @@ sphinx-html:
 	@touch docs/.nojekyll
 	cp -a docsrc/_build/html/. docs
 
+.PHONY:pipenv-setup
+pipenv-setup:devdocker ## Run pipenv-setup to update setup.py with latest dependencies
+	docker compose run --rm --name spark_data_frame_comparer dev sh -c "pipenv run pipenv install --skip-lock --categories \"pipenvsetup\" && pipenv run pipenv-setup sync --pipfile" && \
+	make run-pre-commit
+
 .PHONY:shell
 shell:devdocker ## Brings up the bash shell in dev docker
 	docker compose run --rm --name sfs_shell dev /bin/bash

Pipfile

@@ -44,6 +44,16 @@ lxml=">=4.9.1"
 jinja2=">=3.1.2"
 #jsonpickle=">=2.2.0"
 
+# These dependencies are required for pipenv-setup.  They conflict with ones above, so we install these
+# only when running pipenv-setup
+[pipenvsetup]
+# vistr is needed for visualizing the dependency graph
+vistir=">=0.6.1, <0.7.0" # https://github.com/Madoshakalaka/pipenv-setup/issues/138
+# plete is needed for tab completion
+plette = "<1.0.0" # https://github.com/Madoshakalaka/pipenv-setup/issues/138
+# pipenv-setup is needed for updating setup.py with the dependencies for anyone installing this package
+pipenv-setup = ">=3.2.0"
+
 [requires]
 python_version = "3.12"
 

docker-compose.yml

@@ -8,6 +8,5 @@ services:
     image: sparkfhirschemas_dev
     volumes:
       - ./:/sfs/
-      - ~/.local/share/virtualenvs:/.local/share/virtualenvs:rw
-    container_name: sfs_dev
+    container_name: sparkfhirschemas_dev
     working_dir: /sfs

setup.cfg

@@ -1,7 +1,7 @@
 [tool:pytest]
 addopts =
 [mypy]
-python_version = 3.6
+python_version = 3.12
 warn_return_any = True
 warn_unused_configs = True
 [mypy-lxml.*]

setup.py

@@ -1,5 +1,4 @@
 # noinspection Mypy
-from typing import Any
 
 from setuptools import setup, find_packages
 from os import path, getcwd
@@ -19,33 +18,11 @@
     raise
 
 
-def fix_setuptools() -> None:
-    """Work around bugs in setuptools.
-
-    Some versions of setuptools are broken and raise SandboxViolation for normal
-    operations in a virtualenv. We therefore disable the sandbox to avoid these
-    issues.
-    """
-    try:
-        from setuptools.sandbox import DirectorySandbox
-
-        # noinspection PyUnusedLocal
-        def violation(operation: Any, *args: Any, **_: Any) -> None:
-            print("SandboxViolation: %s" % (args,))
-
-        DirectorySandbox._violation = violation
-    except ImportError:
-        pass
-
-
-# Fix bugs in setuptools.
-fix_setuptools()
-
 # classifiers list is here: https://pypi.org/classifiers/
 
 # create the package setup
 setup(
-    install_requires=["pyspark==3.5.1"],
+    install_requires=["py4j==0.10.9.7", "pyspark==3.5.1"],
     name=package_name,
     version=version,
     author="Imran Qureshi",

spark.Dockerfile

@@ -1,4 +1,4 @@
-FROM imranq2/helix.spark:3.5.1.1-slim
+FROM imranq2/helix.spark:3.5.1.3-slim
 # https://github.com/icanbwell/helix.spark
 USER root
 
@@ -9,26 +9,20 @@ COPY Pipfile* /sfs/
 WORKDIR /sfs
 
 RUN df -h # for space monitoring
-RUN pipenv sync --dev --system && pipenv run pip install pyspark==3.5.1
+RUN pipenv sync --dev --system --extra-pip-args="--prefer-binary"
 
 # override entrypoint to remove extra logging
 RUN mv /opt/minimal_entrypoint.sh /opt/entrypoint.sh
 
 USER root
-# install python 3.12 - it's not available in normal ubuntu repositories
-# https://github.com/deadsnakes/issues/issues/53
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys F23C5A6CF475977595C89F51BA6932366A755776 && \
-    echo "deb https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu/ jammy main" | tee /etc/apt/sources.list.d/deadsnakes-ubuntu-ppa-lunar.list && \
-    apt-get update && apt-get install -y python3.12 && \
-    update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1 && \
-    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
-
-RUN pip install pyspark==3.5.1
-RUN pip install pytest>=8.2.2
 
 COPY . /sfs
 
 RUN df -h # for space monitoring
 RUN mkdir -p /fhir && chmod 777 /fhir
 RUN mkdir -p /.local/share/virtualenvs && chmod 777 /.local/share/virtualenvs
 # USER 1001
+
+# Run as non-root user
+# https://spark.apache.org/docs/latest/running-on-kubernetes.html#user-identity
+USER 185