updated to Spark image 3.5.1.1-slim and to Python 3.12

.github/workflows/build_and_test.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v3
         with:
-          python-version: '3.7'
+          python-version: '3.12'
 
       - name: pre-commit
         run: make run-pre-commit

.github/workflows/python-publish.yml

@@ -17,7 +17,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.7'
+        python-version: '3.12'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.pre-commit-config.yaml

@@ -26,4 +26,4 @@ repos:
         types: [ python ]
         entry: mypy
         require_serial: true
-        args: [ --strict, --python-version=3.7, --show-error-codes, --allow-untyped-decorators ]
+        args: [ --strict, --python-version=3.12, --show-error-codes, --allow-untyped-decorators ]

Makefile

@@ -3,22 +3,24 @@ LANG=en_US.utf-8
 export LANG
 
 Pipfile.lock: Pipfile
-	docker-compose run --rm --name spark_fhir_schemas dev pipenv lock --dev
+	docker compose run --rm --name spark_fhir_schemas dev sh -c "rm -f Pipfile.lock && pipenv lock --dev"
 
 .PHONY:devdocker
 devdocker: ## Builds the docker for dev
-	docker-compose build --no-cache
+	docker compose build --no-cache
 
 .PHONY:init
 init: devdocker up setup-pre-commit  ## Initializes the local developer environment
 
 .PHONY: up
 up: Pipfile.lock
-	docker-compose up --build -d --remove-orphans
+	docker compose up --build -d
 
 .PHONY: down
-down:
-	docker-compose down
+down: ## Brings down all the services in docker-compose
+	export DOCKER_CLIENT_TIMEOUT=300 && export COMPOSE_HTTP_TIMEOUT=300
+	docker compose down --remove-orphans && \
+	docker system prune -f
 
 .PHONY:clean-pre-commit
 clean-pre-commit: ## removes pre-commit hook
@@ -34,51 +36,46 @@ run-pre-commit: setup-pre-commit
 
 .PHONY:update
 update: down Pipfile.lock setup-pre-commit  ## Updates all the packages using Pipfile
-	docker-compose run --rm --name sfs_pipenv dev pipenv sync --dev && \
-	make devdocker && \
-	make pipenv-setup
+	docker compose run --rm --name sfs_pipenv dev pipenv sync --dev && \
+	make devdocker
 
 .PHONY:tests
 tests: up
-	docker-compose run --rm --name sfs_tests dev pytest tests
+	docker compose run --rm --name sfs_tests dev pytest tests
 
 .PHONY: sphinx-html
 sphinx-html:
-	docker-compose run --rm --name spark_fhir_schemas dev make -C docsrc html
+	docker compose run --rm --name spark_fhir_schemas dev make -C docsrc html
 	@echo "copy html to docs... why? https://github.com/sphinx-doc/sphinx/issues/3382#issuecomment-470772316"
 	@rm -rf docs/*
 	@touch docs/.nojekyll
 	cp -a docsrc/_build/html/. docs
 
-.PHONY:pipenv-setup
-pipenv-setup:devdocker ## Brings up the bash shell in dev docker
-	docker-compose run --rm --name sfs_tests dev pipenv-setup sync --pipfile
-
 .PHONY:shell
 shell:devdocker ## Brings up the bash shell in dev docker
-	docker-compose run --rm --name sfs_shell dev /bin/bash
+	docker compose run --rm --name sfs_shell dev /bin/bash
 
 .PHONY:schema-r4
 schema-r4:
-	docker-compose run --rm --name sfs_shell dev python3 spark_fhir_schemas/r4/generate_schema.py && \
+	docker compose run --rm --name sfs_shell dev python3 spark_fhir_schemas/r4/generate_schema.py && \
 	make run-pre-commit
 	make run-pre-commit
 
 .PHONY:schema-r4b
 schema-r4b:
-	docker-compose run --rm --name sfs_shell dev python3 spark_fhir_schemas/r4b/generate_schema.py && \
+	docker compose run --rm --name sfs_shell dev python3 spark_fhir_schemas/r4b/generate_schema.py && \
 	make run-pre-commit
 	make run-pre-commit
 
 .PHONY:schema-stu3
 schema-stu3:
-	docker-compose run --rm --name sfs_shell dev python3 spark_fhir_schemas/stu3/generate_schema.py && \
+	docker compose run --rm --name sfs_shell dev python3 spark_fhir_schemas/stu3/generate_schema.py && \
 	make run-pre-commit
 	make run-pre-commit
 
 .PHONY:schema-dstu2
 schema-dstu2:
-	docker-compose run --rm --name sfs_shell dev python3 spark_fhir_schemas/dstu2/generate_schema_file.py && \
-	docker-compose run --rm --name sfs_shell dev python3 spark_fhir_schemas/dstu2/generate_schema.py && \
+	docker compose run --rm --name sfs_shell dev python3 spark_fhir_schemas/dstu2/generate_schema_file.py && \
+	docker compose run --rm --name sfs_shell dev python3 spark_fhir_schemas/dstu2/generate_schema.py && \
 	make run-pre-commit
 	make run-pre-commit

Pipfile

@@ -4,34 +4,48 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-pyspark = ">=3.1.1"
+# py4j is needed for connecting to the JVM
+py4j="==0.10.9.7" # https://spark.apache.org/docs/latest/api/python/getting_started/install.html#dependencies
+# pyspark is needed for working with Spark
+pyspark="==3.5.1"  # should match the version of spark we use for testing
 
 [dev-packages]
-setuptools=">=60.9.3"
-wheel=">=0.37.1"
-twine=">=3.8.0"
-pre-commit=">=2.20.0"
-autoflake=">=1.5.3"
-mypy = "==0.971"
-pytest = ">=7.1.2"
-black = ">=22.8.0"
-importlib-metadata = ">=4.0.1"
-py4j = "==0.10.9.5" # https://spark.apache.org/docs/latest/api/python/getting_started/install.html#dependencies
-pyspark="==3.3.0"  # should match the version of spark we use for testing
-pygments = ">=2.8.1" # not directly required, pinned by Snyk to avoid a vulnerability
-types-Deprecated=">=0.1.2"
+# setuptools is needed for building the package
+setuptools=">=72.1.0"
+packaging=">=24.1"
+# wheel is needed for building the package
+wheel = ">=0.43.0"
+twine=">=5.1.1"
+# pre-commit is needed for running code quality checks
+pre-commit=">=3.7.1"
+# autoflake is needed for removing unused imports
+autoflake=">=2.3.1"
+# mypy is needed for type checking
+mypy = ">=1.10.1"
+# pytest is needed for running tests
+pytest = ">=8.2.2"
+# black is needed for formatting code
+black = ">=24.4.2"
+# importlib-metadata is needed for working with metadata
+importlib-metadata = ">=5.2.0"
+pygments = ">=2.18.0" # not directly required, pinned by Snyk to avoid a vulnerability
+types-Deprecated=">=1.2.9.20240311"
+# Sphinx is needed for generating documentation
 Sphinx="==4.1.2"
-sphinx-autoapi="==1.8.4"
-sphinx-rtd-theme="==0.5.2"
-myst-parser="==0.15.1"
-pipenv-setup = ">=3.2.0"
-vistir="==0.6.1" # https://github.com/Madoshakalaka/pipenv-setup/issues/138
-lxml="==4.9.1"
-jinja2="==3.1.2"
-jsonpickle="==2.2.0"
+# sphinx-autoapi is needed for generating API documentation
+sphinx-autoapi="==2.0.0"
+# sphinx-rtd-theme is needed for the Read the Docs theme
+sphinx-rtd-theme="==1.0.0"
+# myst-parser is needed for parsing Markdown
+myst-parser="==0.17.2"
+
+vistir=">=0.6.1" # https://github.com/Madoshakalaka/pipenv-setup/issues/138
+lxml=">=4.9.1"
+jinja2=">=3.1.2"
+#jsonpickle=">=2.2.0"
 
 [requires]
-python_version = "3.7"
+python_version = "3.12"
 
 [pipenv]
 allow_prereleases = false