Merge branch 'development' into master

.github/workflows/requirements-update.yml

@@ -0,0 +1,60 @@
+name: Requirements Update
+
+on:
+  schedule:
+    - cron: '0 12 * * 1' # runs at 12:00 UTC on Mondays
+  workflow_dispatch:
+
+jobs:
+
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v4
+        with:
+          only-labels: dependencies,automated pr
+          stale-pr-message: 'This PR is stale because it has been open 7 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
+          close-pr-message: 'This PR was closed because it has been stalled for 7 days with no activity.'
+          days-before-pr-stale: 7
+          days-before-pr-close: 7
+          delete-branch: true
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        ref: development
+
+    - name: Setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.10'
+
+    - name: Install dev Python packages
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r dev-requirements.txt
+
+    - name: Check for pip-tools upgrades
+      run: |
+        pip-compile --generate-hashes \
+          --allow-unsafe \
+          --upgrade \
+          --output-file requirements.txt requirements.in
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v3
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        base: development
+        branch: requirements-updates
+        branch-suffix: timestamp
+        delete-branch: true
+        commit-message: "fix(requirements): Updated Python requirements"
+        title: 'Python Requirements Updates'
+        body: >
+          This PR is auto-generated by Github Actions job [requirements-update].
+        labels: dependencies, automated pr

.github/workflows/scan.yml

@@ -0,0 +1,67 @@
+name: Scan
+
+on:
+  push:
+    branches: [ master, development ]
+  pull_request:
+    branches: [ master, development ]
+  schedule:
+    - cron: '0 12 * * 1' # runs at 12:00 UTC on Mondays
+  workflow_dispatch:
+
+jobs:
+
+  scan:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+    - name: Set image name
+      id: setimagename
+      run: |
+          echo "Image name: $GITHUB_REPOSITORY:$GITHUB_SHA"
+          echo "::set-output name=imagename::$GITHUB_REPOSITORY:$GITHUB_SHA"
+
+    - name: Build the image
+      id: buildimage
+      uses: docker/build-push-action@v2
+      with:
+        context: ./
+        file: ./Dockerfile
+        push: false
+        tags: ${{ steps.setimagename.outputs.imagename }}
+
+    - name: Check whether container scanning should be enabled
+      id: checkcontainerscanning
+      env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+      run: |
+          echo "Enable container scanning: ${{ env.SNYK_TOKEN != '' }}"
+          echo "::set-output name=enabled::${{ env.SNYK_TOKEN != '' }}"
+
+    - name: Run Snyk to check Docker image for vulnerabilities
+      uses: snyk/actions/docker@master
+      if: steps.checkcontainerscanning.outputs.enabled == 'true'
+      continue-on-error: true
+      env:
+        SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+      with:
+        image: ${{ steps.setimagename.outputs.imagename }}
+        args: --file=Dockerfile
+
+    - name: Upload result to GitHub Code Scanning
+      uses: github/codeql-action/upload-sarif@v1
+      if: steps.checkcontainerscanning.outputs.enabled == 'true'
+      with:
+        sarif_file: snyk.sarif

.github/workflows/test.yml

@@ -0,0 +1,44 @@
+name: Test
+
+on:
+  push:
+    branches: [ master, development ]
+  pull_request:
+    branches: [ master, development ]
+    paths:
+      - 'requirements.in'
+      - 'requirements.txt'
+      - 'Dockerfile'
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+    - name: Set image name
+      id: setimagename
+      run: |
+          echo "Image name: $GITHUB_REPOSITORY:$GITHUB_SHA"
+          echo "::set-output name=imagename::$GITHUB_REPOSITORY:$GITHUB_SHA"
+
+    - name: Build the image
+      id: buildimage
+      uses: docker/build-push-action@v2
+      with:
+        context: ./
+        file: ./Dockerfile
+        push: false
+        tags: ${{ steps.setimagename.outputs.imagename }}

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+minimum_pre_commit_version: "2.13.0"
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks.git
+    rev: v4.2.0
+    hooks:
+    - id: trailing-whitespace
+    - id: mixed-line-ending
+    - id: check-byte-order-marker
+    - id: check-merge-conflict
+    - id: detect-aws-credentials
+  - repo: https://github.com/jazzband/pip-tools
+    rev: 6.6.0
+    hooks:
+      - id: pip-compile
+        name: pip-compile dev-requirements.in
+        args: [dev-requirements.in, --upgrade, --generate-hashes, --allow-unsafe, --output-file, dev-requirements.txt]
+        files: ^dev-requirements\.(in|txt)$
+      - id: pip-compile
+        name: pip-compile requirements.in
+        args: [requirements.in, --upgrade, --generate-hashes, --allow-unsafe, --output-file, requirements.txt]
+        files: ^requirements\.(in|txt)$

Dockerfile

@@ -1,36 +1,46 @@
-FROM python:3.6-alpine3.11 AS builder
+FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.0 AS builder
 
-# Install dependencies
-RUN apk add --update \
-    build-base \
-    g++ \
-    libffi-dev \
-    mariadb-dev \
-    git
+# Install requirements
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl \
+        ca-certificates \
+        bzip2 \
+        gcc \
+        default-libmysqlclient-dev \
+        libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 # Add requirements
-ADD requirements /requirements
+ADD requirements.* /
 
-# Use this until we can safely update to Alpine 3.13 or above
-ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
+# Build Python wheels with hash checking
+RUN pip install -U wheel \
+    && pip wheel -r /requirements.txt \
+        --wheel-dir=/root/wheels
 
-# Install Python packages
-RUN pip install -r /requirements/requirements.txt
+FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.0
 
-FROM hmsdbmitc/dbmisvc:alpine-zip-python3.6-0.1.0
+# Copy Python wheels from builder
+COPY --from=builder /root/wheels /root/wheels
 
-RUN apk add --no-cache --update \
-    mariadb-connector-c git libffi-dev git \
-  && rm -rf /var/cache/apk/*
+# Install requirements
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        default-libmysqlclient-dev \
+    && rm -rf /var/lib/apt/lists/*
 
-# Copy pip packages from builder
-COPY --from=builder /root/.cache /root/.cache
+# Add requirements files
+ADD requirements.* /
 
-# Add requirements
-ADD requirements /requirements
-
-# Install Python packages
-RUN pip install -r /requirements/requirements.txt
+# Install Python packages from wheels
+RUN pip install --no-index \
+        --find-links=/root/wheels \
+        --force-reinstall \
+        # Use requirements without hashes to allow using wheels.
+        # For some reason the hashes of the wheels change between stages
+        # and Pip errors out on the mismatches.
+        -r /requirements.in
 
 # Copy app source
 COPY /fileservice /app

buildspec.yml

@@ -25,3 +25,8 @@ phases:
       - echo Build completed on `date`
       - echo Pushing the Docker image...
       - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG
+      - echo Generate imagedefinitions.json file
+      - printf "[{\"name\":\"$CONTAINER_NAME\",\"imageUri\":\"$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG\"}]" > imagedefinitions.json
+
+artifacts:
+    files: imagedefinitions.json

dev-requirements.in

@@ -0,0 +1,2 @@
+pip-tools
+pre-commit

dev-requirements.txt

@@ -0,0 +1,111 @@
+#
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
+#
+#    pip-compile --allow-unsafe --generate-hashes --output-file=dev-requirements.txt dev-requirements.in
+#
+cfgv==3.3.1 \
+    --hash=sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426 \
+    --hash=sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736
+    # via pre-commit
+click==8.1.3 \
+    --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \
+    --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48
+    # via pip-tools
+distlib==0.3.4 \
+    --hash=sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b \
+    --hash=sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579
+    # via virtualenv
+filelock==3.6.0 \
+    --hash=sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85 \
+    --hash=sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0
+    # via virtualenv
+identify==2.5.0 \
+    --hash=sha256:3acfe15a96e4272b4ec5662ee3e231ceba976ef63fd9980ed2ce9cc415df393f \
+    --hash=sha256:c83af514ea50bf2be2c4a3f2fb349442b59dc87284558ae9ff54191bff3541d2
+    # via pre-commit
+nodeenv==1.6.0 \
+    --hash=sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b \
+    --hash=sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7
+    # via pre-commit
+pep517==0.12.0 \
+    --hash=sha256:931378d93d11b298cf511dd634cf5ea4cb249a28ef84160b3247ee9afb4e8ab0 \
+    --hash=sha256:dd884c326898e2c6e11f9e0b64940606a93eb10ea022a2e067959f3a110cf161
+    # via pip-tools
+pip-tools==6.6.0 \
+    --hash=sha256:66318bc2e884b61fafa1cb2cf01b35fdd779ab9ce82cc1bce277adb8cf3ab845 \
+    --hash=sha256:98aa24004440a1c0489d71a567a4e8afdf23c7782bff483d1219881e7302de83
+    # via -r dev-requirements.in
+platformdirs==2.5.2 \
+    --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \
+    --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19
+    # via virtualenv
+pre-commit==2.19.0 \
+    --hash=sha256:10c62741aa5704faea2ad69cb550ca78082efe5697d6f04e5710c3c229afdd10 \
+    --hash=sha256:4233a1e38621c87d9dda9808c6606d7e7ba0e087cd56d3fe03202a01d2919615
+    # via -r dev-requirements.in
+pyyaml==6.0 \
+    --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \
+    --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \
+    --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \
+    --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \
+    --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \
+    --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \
+    --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \
+    --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \
+    --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \
+    --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \
+    --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \
+    --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \
+    --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \
+    --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \
+    --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \
+    --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \
+    --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \
+    --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \
+    --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \
+    --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \
+    --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \
+    --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \
+    --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \
+    --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \
+    --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \
+    --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \
+    --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \
+    --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \
+    --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \
+    --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \
+    --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \
+    --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \
+    --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5
+    # via pre-commit
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
+    # via virtualenv
+toml==0.10.2 \
+    --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
+    --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
+    # via pre-commit
+tomli==2.0.1 \
+    --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
+    --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
+    # via pep517
+virtualenv==20.14.1 \
+    --hash=sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a \
+    --hash=sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5
+    # via pre-commit
+wheel==0.37.1 \
+    --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \
+    --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4
+    # via pip-tools
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==22.0.4 \
+    --hash=sha256:b3a9de2c6ef801e9247d1527a4b16f92f2cc141cd1489f3fffaf6a9e96729764 \
+    --hash=sha256:c6aca0f2f081363f689f041d90dab2a07a9a07fb840284db2218117a52da800b
+    # via pip-tools
+setuptools==62.1.0 \
+    --hash=sha256:26ead7d1f93efc0f8c804d9fafafbe4a44b179580a7105754b245155f9af05a8 \
+    --hash=sha256:47c7b0c0f8fc10eec4cf1e71c6fdadf8decaa74ffa087e68cd1c20db7ad6a592
+    # via pip-tools