Merge pull request #34 from icanbwell/switch_to_docker

switch to docker

.github/workflows/build_and_test.yml

@@ -14,27 +14,34 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      # Checks-out your repository
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v3
         with:
-          python-version: '3.6.11'
-      - uses: actions/setup-java@v1
-        with:
-          java-version: '11.0.8' # The JDK version to make available on the path.
-          java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
-          architecture: x64 # (x64 or x86) - defaults to x64
-    # install required tools
-      - name: prereqs
+          python-version: '3.7'
+
+      - name: Set cache permissions
         run: |
-          pip install wheel
+          mkdir ~/.local && mkdir ~/.local/share && mkdir ~/.local/share/virtualenvs
+          chown -R $(whoami) ~/.local/share/virtualenvs/
 
-      # setup and install everything and run tests
-      - name: install_spark
-        run: make download_spark
-      - name: setup_packages
-        run: make setup_packages
-      - name: run-pre-commit
+      - name: Cache pip dependencies
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-pip-deps-warehouse-p
+        with:
+          path: ~/.local/share/virtualenvs/
+          key: ${{ runner.os }}-${{ env.CACHE_SEED }}-${{ github.job }}-${{ env.cache-name }}-${{ hashFiles('Pipfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ env.CACHE_SEED }}-${{ github.job }}-${{ env.cache-name }}-
+            ${{ runner.os }}-${{ env.CACHE_SEED }}-${{ github.job }}-
+            ${{ runner.os }}-${{ env.CACHE_SEED }}-
+            ${{ runner.os }}-
+
+      - name: pre-commit
         run: make run-pre-commit
-      - name: continuous_integration
-        run: make continuous_integration
+
+      # setup and install everything and run tests
+      - name: up
+        run: make up
+      - name: tests
+        run: make tests

.github/workflows/python-publish.yml

@@ -17,11 +17,15 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.6'
+        python-version: '3.7'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install setuptools wheel twine
+    - name: Sync Pipfile with setup.py
+      run: |
+        python -m pip install --no-cache-dir pipenv
+        pipenv sync --dev --system
     - name: Get Previous tag
       id: previoustag
       uses: WyriHaximus/github-action-get-previous-tag@master
@@ -31,7 +35,7 @@ jobs:
         path: VERSION
         contents: |
           ${{ steps.previoustag.outputs.tag }}
-        write-mode: overwrite 
+        write-mode: overwrite
     - name: Build and publish
       env:
         TWINE_USERNAME: __token__

.pre-commit-config.yaml

@@ -1,31 +1,36 @@
-exclude: ^.idea/
+exclude: (^.idea/|^notebooks/)
 repos:
--   repo: https://github.com/humitos/mirrors-autoflake
-    rev: v1.3
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
     hooks:
-    -   id: autoflake
-        args: [--in-place, --remove-all-unused-imports, --remove-unused-variable]
--   repo: https://github.com/hadialqattan/pycln
-    rev: v0.0.2 # Possible releases: https://github.com/hadialqattan/pycln/tags
+      - id: end-of-file-fixer
+        types: [ python ]
+  - repo: local
     hooks:
-    -   id: pycln
-        args: [--all]
--   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
-    hooks:
-    -   id: flake8
--   repo: https://github.com/ambv/black
-    rev: 21.5b0
-    hooks:
-    -   id: black
-        language_version: python3.6
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.790
-    hooks:
-    -   id: mypy
-        args: [--strict, --python-version=3.6, --show-error-codes, --allow-untyped-decorators]
-        additional_dependencies:
-            - numpy>=1.19.2
-            - pyspark==3.1.1
-            - logger
-            - pytest
+      - id: autoflake
+        name: autoflake
+        language: system
+        types: [ python ]
+        entry: autoflake
+        require_serial: true
+        args: [ '--in-place', '--remove-all-unused-imports', '--ignore-init-module-imports']
+      - id: flake8
+        name: flake8
+        language: system
+        types: [python]
+        entry: flake8
+        require_serial: true
+        args: [ "--per-file-ignores=__init__.py:F401,anonymizer.py:E731"]
+      - id: black
+        name: black
+        language: system
+        types: [python]
+        require_serial: true
+        entry: black
+      - id: mypy
+        name: mypy
+        language: system
+        types: [ python ]
+        entry: mypy
+        require_serial: true
+        args: [ --strict, --python-version=3.7, --show-error-codes, --allow-untyped-decorators ]

Makefile

@@ -2,108 +2,58 @@ LANG=en_US.utf-8
 
 export LANG
 
-BRANCH=$(shell git rev-parse --abbrev-ref HEAD)
-VERSION=$(shell cat VERSION)
-VENV_NAME=venv
-GIT_HASH=${CIRCLE_SHA1}
-SPARK_VER=3.1.1
-HADOOP_VER=3.2
-PACKAGES_FOLDER=venv/lib/python3.6/site-packages
-SPF_BASE=${PACKAGES_FOLDER}
+Pipfile.lock: Pipfile
+	docker-compose run --rm --name spark_fhir_schemas dev pipenv lock --dev
 
-include spark_fhir_schemas/Makefile.spark
-include spark_fhir_schemas/Makefile.docker
-include spark_fhir_schemas/Makefile.python
+.PHONY:devdocker
+devdocker: ## Builds the docker for dev
+	docker-compose build --no-cache
 
-.PHONY:devsetup
-devsetup:venv
-	. $(VENV_NAME)/bin/activate && \
-    pip install --upgrade pip && \
-    pip install --upgrade -r requirements.txt && \
-    pip install --upgrade -r requirements-test.txt && \
-    pre-commit install && \
-    python setup.py install
-
-.PHONY:checks
-checks:venv
-	. $(VENV_NAME)/bin/activate && \
-    pip install --upgrade -r requirements.txt && \
-    flake8 spark_fhir_schemas && \
-    mypy spark_fhir_schemas --strict && \
-    flake8 tests && \
-    mypy tests --strict
-
-.PHONY:update
-update:
-	. $(VENV_NAME)/bin/activate && \
-	pip install --upgrade -r requirements.txt && \
-	pip install --upgrade -r requirements-test.txt
-
-.PHONY:build
-build:venv
-	. $(VENV_NAME)/bin/activate && \
-    pip install --upgrade pip && \
-    pip install --upgrade -r requirements.txt && \
-    python setup.py install && \
-    rm -r dist/ && \
-    python3 setup.py sdist bdist_wheel
-
-.PHONY:testpackage
-testpackage:venv build
-	. $(VENV_NAME)/bin/activate && \
-	python3 -m twine upload -u __token__ --repository testpypi dist/*
-# password can be set in TWINE_PASSWORD. https://twine.readthedocs.io/en/latest/
+.PHONY:init
+init: devdocker up setup-pre-commit  ## Initializes the local developer environment
 
-.PHONY:package
-package:venv build
-	. $(VENV_NAME)/bin/activate && \
-	python3 -m twine upload -u __token__ --repository pypi dist/*
-# password can be set in TWINE_PASSWORD. https://twine.readthedocs.io/en/latest/
+.PHONY: up
+up: Pipfile.lock
+	docker-compose up --build -d --remove-orphans
 
-.PHONY:tests
-tests:
-	. $(VENV_NAME)/bin/activate && \
-    pip install --upgrade -r requirements.txt && \
-	pip install --upgrade -r requirements-test.txt && \
-	pytest tests
+.PHONY: down
+down:
+	docker-compose down
 
 .PHONY:clean-pre-commit
-clean-pre-commit:
-	pre-commit clean
+clean-pre-commit: ## removes pre-commit hook
+	rm -f .git/hooks/pre-commit
 
 .PHONY:setup-pre-commit
-setup-pre-commit:
-	pre-commit install
+setup-pre-commit: Pipfile.lock
+	cp ./pre-commit-hook ./.git/hooks/pre-commit
 
 .PHONY:run-pre-commit
-run-pre-commit:
-	pre-commit run --all-files
+run-pre-commit: setup-pre-commit
+	./.git/hooks/pre-commit
 
-.PHONY:init
-init: installspark up devsetup tests
-
-.PHONY:schema
-schema:
-	. ${VENV_NAME}/bin/activate && \
-	python3 spark_fhir_schemas/r4/generate_schema.py && \
-	pre-commit run --all-files
-	pre-commit run --all-files
-
-.PHONY:schema-stu3
-schema-stu3:
-	. ${VENV_NAME}/bin/activate && \
-	python3 spark_fhir_schemas/stu3/generate_schema.py && \
-	pre-commit run --all-files
-	pre-commit run --all-files
-# run twice to check that the formatting was fixed
-
-.PHONY:setup_packages
-setup_packages:
-	pip install --upgrade pip && \
-    pip install --upgrade -r requirements.txt && \
-    pip install --upgrade -r requirements-test.txt && \
-    python setup.py install
+.PHONY:update
+update: down Pipfile.lock setup-pre-commit  ## Updates all the packages using Pipfile
+	docker-compose run --rm --name sfs_pipenv dev pipenv sync --dev && \
+	make devdocker && \
+	make pipenv-setup
 
-.PHONY:continuous_integration
-continuous_integration:
-	pytest tests
+.PHONY:tests
+tests: up
+	docker-compose run --rm --name sfs_tests dev pytest tests
+
+.PHONY: sphinx-html
+sphinx-html:
+	docker-compose run --rm --name spark_fhir_schemas dev make -C docsrc html
+	@echo "copy html to docs... why? https://github.com/sphinx-doc/sphinx/issues/3382#issuecomment-470772316"
+	@rm -rf docs/*
+	@touch docs/.nojekyll
+	cp -a docsrc/_build/html/. docs
+
+.PHONY:pipenv-setup
+pipenv-setup:devdocker ## Brings up the bash shell in dev docker
+	docker-compose run --rm --name sfs_tests dev pipenv-setup sync --pipfile
+
+.PHONY:shell
+shell:devdocker ## Brings up the bash shell in dev docker
+	docker-compose run --rm --name sfs_shell dev /bin/bash

Pipfile

@@ -0,0 +1,34 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pyspark = "==3.1.1"
+
+[dev-packages]
+setuptools=">=60.9.3"
+wheel=">=0.37.1"
+twine=">=3.8.0"
+pre-commit=">=2.17.0"
+autoflake=">=1.4"
+flake8 = ">=4.0.1"
+mypy = ">=0.931"
+pytest = ">=7.0.1"
+black = ">=22.1.0"
+importlib-metadata = ">=4.0.1"
+py4j = "==0.10.9"
+pyspark="==3.1.1"  # should match the version of spark we use for testing
+pygments = ">=2.8.1" # not directly required, pinned by Snyk to avoid a vulnerability
+types-Deprecated=">=0.1.2"
+Sphinx="==4.1.2"
+sphinx-autoapi="==1.8.4"
+sphinx-rtd-theme="==0.5.2"
+myst-parser="==0.15.1"
+pipenv-setup = ">=3.2.0"
+
+[requires]
+python_version = "3.7"
+
+[pipenv]
+allow_prereleases = false