Use 'docker compose' to run tests and linters

AlmaLinux · Apr 3, 2024 · c8f3b78 · c8f3b78
1 parent 2858e6c
commit c8f3b78
Show file tree

Hide file tree

Showing 13 changed files with 224 additions and 137 deletions.
diff --git a/.coveragerc b/.coveragerc
diff --git a/.github/workflows/preflight-summary.yml b/.github/workflows/preflight-summary.yml
@@ -53,7 +53,13 @@ jobs:
             printf "\n\n### Code Coverage Summary\n"
             cat code-coverage-results.md
             JOB_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/${{ github.event.workflow_run.id }}"
-            printf "\nView full reports on the [Job Summary]($JOB_URL \"Go to Job Summary\") page\n"
+            printf "\nView full reports on the [Job Summary]($JOB_URL \"Go to Job Summary\") page\n\n"
+
+            cat {pylint,black,isort,bandit}-report.md > linter-reports.md 2>/dev/null || true
+            if [[ -s linter-reports.md ]]; then
+              printf "### Linter reports\n"
+              cat linter-reports.md
+            fi
           } > preflight-report.md
 
       - name: Comment PR

diff --git a/.github/workflows/preflight.yml b/.github/workflows/preflight.yml
@@ -32,7 +32,7 @@ jobs:
             exit 2
           fi
 
-  pytest:
+  preflight:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     env:
@@ -42,26 +42,91 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
 
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v5
+      - name: Set up Docker Buildx
+        # https://github.com/marketplace/actions/docker-setup-buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        # https://github.com/marketplace/actions/build-and-push-docker-images
+        uses: docker/build-push-action@v5
         with:
-          python-version: 3.9
-          cache: pip
-          cache-dependency-path: setup.py
+          context: .
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
-      - name: Install dependencies
-        run: pip install -e .
+      - name: Prepare working directory
+        run: |
+          mkdir -p $REPORTS_DIR
+          touch .env
 
-      - name: Install Pytest
-        run: pip install pytest pytest-cov
+      - name: Get changed .py files
+        # https://github.com/marketplace/actions/paths-changes-filter
+        uses: dorny/paths-filter@v3
+        id: changed-files
+        with:
+          list-files: shell
+          filters: |
+            py:
+              - added|modified: '**/*.py'
+            src:
+              - added|modified: 'sign/**/*.py'
+              - added|modified: '*.py'
+
+      - name: Run pytest
+        run: |
+          docker compose run --rm sign_file bash -c "
+            pytest -v --cov \
+              --junit-xml=$REPORTS_DIR/pytest-report.xml \
+              --cov-report=xml:$REPORTS_DIR/pytest-coverage.xml \
+              --cov-report=term | tee $REPORTS_DIR/pytest-output.txt"
+
+      - name: Run pylint
+        if: ${{ steps.changed-files.outputs.src == 'true' }}
+        run: |
+          docker compose run --rm sign_file bash -c "
+            pylint --exit-zero ${{ steps.changed-files.outputs.src_files }} \
+              | tee $REPORTS_DIR/pylint-report.txt"
 
-      - name: Run Pytest
+      - name: Run black
+        if: ${{ steps.changed-files.outputs.py == 'true' }}
         run: |
-          mkdir -p $REPORTS_DIR
-          pytest -v --cov \
-                --junit-xml=$REPORTS_DIR/pytest-report.xml \
-                --cov-report=xml:$REPORTS_DIR/pytest-coverage.xml \
-                --cov-report=term | tee $REPORTS_DIR/pytest-report.txt || true
+          docker compose run --rm sign_file bash -c "
+            black --check --diff --color ${{ steps.changed-files.outputs.py_files }} \
+              | tee >(sed 's/\x1B\[[0-9;]*m//g' > $REPORTS_DIR/black-report.txt)"
+
+      - name: Run isort
+        if: ${{ steps.changed-files.outputs.py == 'true' }}
+        run: |
+          docker compose run --rm sign_file bash -c "
+            isort --check-only --diff --color ${{ steps.changed-files.outputs.py_files }} \
+              | tee >(sed 's/\x1B\[[0-9;]*m//g' > $REPORTS_DIR/isort-report.txt)"
+
+      - name: Run bandit
+        if: ${{ steps.changed-files.outputs.src == 'true' }}
+        run: |
+          docker compose run --rm sign_file bash -c "
+            bandit -c pyproject.toml ${{ steps.changed-files.outputs.src_files }} \
+              | tee >(sed 's/\x1B\[[0-9;]*m//g' > $REPORTS_DIR/bandit-report.txt)"
+
+      - name: Generate .md reports
+        run: |
+          awk 'NR == 1 {next}; /^-+ coverage:/ {exit}; {print}' $REPORTS_DIR/pytest-output.txt \
+            > $REPORTS_DIR/pytest-report.txt
+          awk '/^-+ coverage:/, /^TOTAL/' $REPORTS_DIR/pytest-output.txt \
+            > $REPORTS_DIR/coverage-report.txt
+
+          for tool in coverage pytest pylint black isort bandit; do
+            if [[ -s $REPORTS_DIR/${tool}-report.txt ]]; then
+              {
+                printf "<details><summary>${tool^} report</summary>\n"
+                printf '\n```\n'
+                cat $REPORTS_DIR/${tool}-report.txt
+                printf '\n```\n'
+                printf '\n</details>\n\n'
+              } > $REPORTS_DIR/${tool}-report.md
+            fi
+          done
 
       - name: Save environment
         run: |
@@ -79,16 +144,5 @@ jobs:
 
       - name: Publish Job Summary
         run: |
-          {
-            printf "<details><summary>Coverage report</summary>\n"
-            printf '\n```\n'
-            awk '/^-+ coverage:/, /^TOTAL/' $REPORTS_DIR/pytest-report.txt
-            printf '\n```\n'
-            printf '</details>\n\n'
-
-            printf '<details><summary>Pytest report</summary>\n'
-            printf '\n```\n'
-            awk 'NR == 1 {next}; /^-+ coverage:/ {exit}; {print}' $REPORTS_DIR/pytest-report.txt
-            printf '\n```\n'
-            printf '</details>\n\n'
-          } > $GITHUB_STEP_SUMMARY
+          cat $REPORTS_DIR/{coverage,pytest,pylint,black,isort,bandit}-report.md \
+            > $GITHUB_STEP_SUMMARY 2>/dev/null || true
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -19,28 +19,31 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
 
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.9
-          cache: pip
-          cache-dependency-path: setup.py
-
-      - name: Install dependencies
-        run: pip install -e .
+      - name: Set up Docker Buildx
+        # https://github.com/marketplace/actions/docker-setup-buildx
+        uses: docker/setup-buildx-action@v3
 
-      - name: Install Pytest
-        run: pip install pytest pytest-cov
+      - name: Build Docker image
+        # https://github.com/marketplace/actions/build-and-push-docker-images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
       - name: Prepare working directory
-        run: mkdir -p $REPORTS_DIR
+        run: |
+          mkdir -p $REPORTS_DIR
+          touch .env
 
-      - name: Run Pytest
+      - name: Run pytest
         id: pytest
         run: |
-          pytest -v --cov \
-            --cov-report=json:$REPORTS_DIR/pytest-report.json \
-            --cov-report=term | tee $REPORTS_DIR/pytest-report.txt || true
+          docker compose run --rm sign_file bash -c "
+            pytest -v --cov \
+              --cov-report=json:$REPORTS_DIR/pytest-report.json \
+              --cov-report=term | tee $REPORTS_DIR/pytest-output.txt"
 
           python -c "
           import json
@@ -61,18 +64,26 @@ jobs:
            maxColorRange: 60
            namedLogo: pytest
 
-      - name: Publish Job Summary
+      - name: Generate .md reports
         run: |
-          {
-            printf "<details><summary>Coverage report</summary>\n"
-            printf '\n```\n'
-            awk '/^-+ coverage:/, /^TOTAL/' $REPORTS_DIR/pytest-report.txt
-            printf '\n```\n'
-            printf '</details>\n\n'
+          awk 'NR == 1 {next}; /^-+ coverage:/ {exit}; {print}' $REPORTS_DIR/pytest-output.txt \
+            > $REPORTS_DIR/pytest-report.txt
+          awk '/^-+ coverage:/, /^TOTAL/' $REPORTS_DIR/pytest-output.txt \
+            > $REPORTS_DIR/coverage-report.txt
+
+          for tool in coverage pytest; do
+            if [[ -s $REPORTS_DIR/${tool}-report.txt ]]; then
+              {
+                printf "<details><summary>${tool^} report</summary>\n"
+                printf '\n```\n'
+                cat $REPORTS_DIR/${tool}-report.txt
+                printf '\n```\n'
+                printf '\n</details>\n\n'
+              } > $REPORTS_DIR/${tool}-report.md
+            fi
+          done
 
-            printf '<details><summary>Pytest report</summary>\n'
-            printf '\n```\n'
-            awk 'NR == 1 {next}; /^-+ coverage:/ {exit}; {print}' $REPORTS_DIR/pytest-report.txt
-            printf '\n```\n'
-            printf '</details>\n\n'
-          } > $GITHUB_STEP_SUMMARY
+      - name: Publish Job Summary
+        run: |
+          cat $REPORTS_DIR/{coverage,pytest}-report.md \
+            > $GITHUB_STEP_SUMMARY 2>/dev/null || true
diff --git a/Dockerfile b/Dockerfile
@@ -1,25 +1,17 @@
-FROM almalinux:8
-
-RUN dnf install -y epel-release && \
-    dnf upgrade -y && \
-    dnf install -y --enablerepo="powertools" --enablerepo="epel" \
-        python39 python39-devel python3-virtualenv python39-setuptools pinentry gnupg2  && \
-    dnf clean all
-
-
-RUN groupadd -g 1000 alt && \
-    useradd -ms /bin/bash -u 1000 -g 1000 alt && \
-    usermod -aG wheel alt && \
-    echo 'alt ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
-    echo 'wheel ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+FROM almalinux:9
 
+RUN <<EOT
+  set -ex
+  dnf upgrade -y
+  dnf install -y pinentry
+  dnf clean all
+EOT
 
 WORKDIR /app
-RUN virtualenv -p python3.9 env
-COPY setup.py /app
-RUN /app/env/bin/pip install -e /app/.
-
-RUN chown -R alt:alt /app
-USER alt
-
-CMD ["/app/env/bin/python", "/app/start.py"]
+COPY requirements.* .
+RUN <<EOT
+  set -ex
+  python3 -m ensurepip
+  pip3 install -r requirements.devel.txt
+  rm requirements.*
+EOT
diff --git a/README.md b/README.md
@@ -12,20 +12,20 @@ Service for signing various text files using PGP
 ### Prerequirements
 1. GnuPGP binary
 2. At least one private RSA key in PGP database
-3. Python3.9 with virtualenv
+3. Python 3.9
 
 ### Install service
 1. Create virtual enviroment
     ```bash
-    virtualenv -p python3.9 env
+    python3 -m venv .venv
     ```
 2. Activate virtualenv
     ```bash
-    source env/bin/activate
+    source .venv/bin/activate
     ```
-3. Install service with pip
+3. Install packages
     ```bash
-    (env) python -m pip install .
+    (.venv) pip3 install -r requirements.devel.txt
     ```
 
 ### Service configuration
@@ -37,8 +37,8 @@ EXAMPLE
 SF_GPG_BINARY="/usr/bin/gpg2"
 
 # SF_KEYRING - path to keyring kbx file
-# default /home/alt/.gnupg/pubring.kbx
-SF_KEYRING="/home/alt/.gnupg/pubring.kbx"
+# default ~/.gnupg/pubring.kbx
+SF_KEYRING="~/.gnupg/pubring.kbx"
 
 # SF_MAX_UPLOAD_BYTES - max file size (in bytes )to sign
 # default 100000000
@@ -82,7 +82,7 @@ SF_DB_URL="sqlite:///./sign-file.sqlite3"
 # SF_HOST_GNUPG -  path of .gnupg directory on host
 # this variable only used in docker-compose file
 # default ""
-SF_HOST_GNUPG="/home/kzhukov/.gnupg"
+SF_HOST_GNUPG="~/.gnupg"
 
 # SF_ROOT_URL root URL for API calls
 # default ""
@@ -97,10 +97,10 @@ SF_ROOT_URL=""
 ### Database initialization
 Create database and user with `db_manage.py` script
    ```bash
-    (env) python db_manage.py create
+    (.venv) python3 db_manage.py create
     command executed succesfully
 
-    (env) python db_manage.py user_add
+    (.venv) python3 db_manage.py user_add
     email:kzhukov@cloudlinux.com
     password:
     password (repeat):
@@ -113,7 +113,7 @@ Create database and user with `db_manage.py` script
 Start service using `startup.py` script
 
 ```bash
-(env) % python start.py
+(.venv) % python3 start.py
 INFO:     Will watch for changes in these directories: ['/Users/kzhukov/projects/cloudlinux/albs-sign-file']
 INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
 DEBUG:    WatchGodReload detected a new excluded dir '.pytest_cache' in '/Users/kzhukov/projects/cloudlinux/albs-sign-file'; Adding to exclude list.

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,20 +1,18 @@
-version: "3.9"
 services:
   sign_file:
-    env_file:
-      - .env
     image: sign-file:latest
     build:
-      dockerfile: Dockerfile
       context: .
-    command: >  
-      bash -c "/app/env/bin/python /app/db_manage.py dev_init 
-      && /app/env/bin/python /app/start.py"
-    restart: on-failure
+    # We could use:
+    #  env_file:
+    #    - path: "./.env"
+    #      required: false
+    # but it doesn't work on GitHub currently
+    # https://github.com/docker/compose/issues/11591
+    env_file: "./.env"
+    command: "bash -c 'python3 db_manage.py dev_init && python3 start.py'"
     volumes:
-      - "${SF_HOST_GNUPG}:/home/alt/.gnupg"
-      - ./sign:/app/sign
-      - ./start.py:/app/start.py
-      - ./db_manage.py:/app/db_manage.py
+      - "${SF_HOST_GNUPG:-./.gnupg}:/root/.gnupg"
+      - ".:/app"
     ports:
       - "8000:8000"