chore: Various updates

.github/workflows/ci.yml

@@ -3,8 +3,6 @@ name: Checks
 on:
   pull_request:
     branches: ["**"]
-  push:
-    branches: ["main"]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -15,19 +13,14 @@ jobs:
     name: Python
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.4
+      - uses: actions/checkout@v4.1.7
+      - name: Install Poetry Action
+        uses: snok/install-poetry@v1.3.4
       - name: Set up Python 3.10
         uses: actions/setup-python@v5.1.0
         with:
           python-version: "3.10"
-      - name: Install Poetry Action
-        uses: snok/install-poetry@v1.3.4
-      - name: Set up Virtualenv Cache
-        uses: actions/cache@v4.0.2
-        with:
-          path: .venv
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
-          restore-keys: ${{ runner.os }}-poetry-
+          cache: "poetry"
       - name: Install Python Dependencies
         run: |
           poetry check --lock
@@ -36,73 +29,41 @@ jobs:
         run: |
           poetry run ruff format . --check
           poetry run ruff check .
-      - name: Set up mypy cache
-        uses: actions/cache@v4.0.2
-        with:
-          path: .mypy_cache
-          key: ${{ runner.os }}-mypy-${{ hashFiles('glueetl/**/*.py') }}
-          restore-keys: ${{ runner.os }}-mypy-
       - name: Check type annotations using mypy
         run: poetry run mypy
-      - name: Check if requirements are up-to-date
+      - name: Check imports using import-linter
+        run: poetry run lint-imports
+      - name: Check if requirements.txt is in sync with poetry.lock
         run: poetry export --with=dev --output docker/requirements.txt && git diff --exit-code
-      - name: Set up pre-commit cache
-        uses: actions/cache@v4.0.2
-        with:
-          path: ~/.cache/pre-commit
-          key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
-          restore-keys: ${{ runner.os }}-pre-commit-
-      - name: Run pre-commit hooks
-        run: |
-          cp docker/.env.sample docker/.env
-          poetry run pre-commit run --all-files
-        env:
-          # Skip redundant hooks that are already covered elsewhere in
-          # this Github workflow
-          SKIP: poetry-check,poetry-lock,poetry-install,lint-python,format-python,typecheck-python,update-requirements,hadolint-docker,shellcheck,checkmake
       - name: Set up NodeJS
         uses: actions/setup-node@v4.0.2
         with:
           node-version: "20"
-      - name: Install NodeJS Dependencies
-        run: npm install -g aws-cdk
       - name: Synthesize CDK deployment package
-        run: cdk synth --strict
+        run: npx aws-cdk synth --strict
   tests:
     name: Run tests in Glue container
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.4
-      - name: Run the checks inside Docker
-        uses: addnab/docker-run-action@v3
-        with:
-          image: amazon/aws-glue-libs:glue_libs_4.0.0_image_01
-          options: --volume ${{ github.workspace }}:/home/glue_user/workspace --env AWS_REGION=us-east-1
-          run: |
-            pip3 install --no-warn-script-location --user --upgrade pip==24.0
-            pip3 install --no-warn-script-location --user -r docker/requirements.txt
-            python3 -m pytest -p no:cacheprovider
-  pip-audit:
-    name: Audit requirements
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4.1.4
-      - name: Audit requirements using pip-audit
-        uses: pypa/gh-action-pip-audit@v1.0.8
-        with:
-          inputs: docker/requirements.txt
+      - uses: actions/checkout@v4.1.7
+      - name: Pull official glue image
+        run: docker pull amazon/aws-glue-libs:glue_libs_4.0.0_image_01
+      - name: Build container
+        run: USER_ID=$(id -u) docker compose --file docker/docker-compose.yml build glue
+      - name: Run tests in container
+        run: USER_ID=$(id -u) docker compose --file docker/docker-compose.yml run glue -c pytest
   shell:
     name: Shell
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.4
+      - uses: actions/checkout@v4.1.7
       - name: Check shell scripts using shellcheck
         uses: ludeeus/action-shellcheck@2.0.0
   dockerfile:
     name: Dockerfile
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.4
+      - uses: actions/checkout@v4.1.7
       - name: Validate Dockerfile using hadolint
         uses: hadolint/hadolint-action@v3.1.0
         with:
@@ -111,6 +72,6 @@ jobs:
     name: Makefile
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.4
+      - uses: actions/checkout@v4.1.7
       - name: Check Makefile using checkmake
         uses: Uno-Takashi/checkmake-action@v2

.pre-commit-config.yaml

@@ -31,56 +31,20 @@ repos:
         pass_filenames: false
         stages: [post-checkout, post-merge]
         always_run: true
+      - id: format-python
+        name: Format Python code
+        language: system
+        types: [python]
+        entry: poetry run ruff format
       - id: lint-python
         name: Lint Python code
         language: system
         types: [python]
         entry: poetry run ruff check
         args: [--fix, --unsafe-fixes]
-      - id: format-python
-        name: Format Python code
-        language: system
-        types: [python]
-        entry: poetry run ruff format
       - id: update-requirements
         name: Update requirements.txt
         language: system
         files: poetry.lock
         entry: poetry export --with=dev --output docker/requirements.txt
         pass_filenames: false
-  - repo: https://github.com/hadolint/hadolint
-    rev: v2.13.0-beta
-    hooks:
-      - id: hadolint-docker
-  - repo: https://github.com/iamthefij/docker-pre-commit
-    rev: v3.0.1
-    hooks:
-      - id: docker-compose-check
-  - repo: https://github.com/koalaman/shellcheck-precommit
-    rev: v0.10.0
-    hooks:
-      - id: shellcheck
-        name: Check shell scripts
-  - repo: https://github.com/Yelp/detect-secrets
-    rev: v1.4.0
-    hooks:
-      - id: detect-secrets
-  - repo: https://github.com/rhysd/actionlint
-    rev: v1.6.27
-    hooks:
-      - id: actionlint-docker
-  - repo: https://github.com/mrtazz/checkmake
-    rev: 0.2.2
-    hooks:
-      - id: checkmake
-        name: Check Makefile syntax
-  - repo: https://github.com/kynan/nbstripout
-    rev: 0.7.1
-    hooks:
-      - id: nbstripout
-        name: Strip output from Jupyter notebooks
-  - repo: https://github.com/seddonym/import-linter
-    rev: v2.0
-    hooks:
-      - id: import-linter
-        name: Check for violations of import rules

docker/Dockerfile

@@ -10,22 +10,24 @@ ARG USER_ID=1000
 # Switch to root to be able to make changes in the container filesystem.
 USER root
 
-# Change UID of glue_user to be the same as host user. This allows
-# JupyterLab to write to the host system as glue_user.
-RUN usermod -u $USER_ID glue_user \
-  # Clean up /tmp which may already have glue_user-owned files with the
-  # old UID.
-  && rm -rf /tmp/*
+# Clean up /tmp which may already have glue_user-owned files with the
+# old UID.
+RUN rm -rf /tmp/* \
+  # Change UID of glue_user to be the same as host user. This allows
+  # JupyterLab to write to the host system as glue_user.
+  && usermod -u $USER_ID glue_user \
+  && mkdir -p docker
 
 # Switch to glue_user to be able to make changes for the user itself.
 USER glue_user
 
 # Copy requirements file that contains tooling.
 WORKDIR /home/glue_user/workspace
-COPY docker/requirements.txt requirements.txt
+COPY docker/requirements.txt docker/requirements.txt
 
-RUN pip3 install --no-cache-dir --no-warn-script-location --user --upgrade pip==24.0 \
+# hadolint ignore=DL3013
+RUN pip3 install --no-cache-dir --no-warn-script-location --user --upgrade pip \
   # Install dev requirements.
-  && pip3 install --no-cache-dir --no-warn-script-location --user -r requirements.txt  \
+  && pip3 install --no-cache-dir --no-warn-script-location --user -r docker/requirements.txt  \
   # Prepare a /tmp directory needed by Spark to start.
   && mkdir -p /tmp/spark-events

docker/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.8"
-
 services:
   glue:
     container_name: glue-devtools
@@ -19,7 +17,8 @@ services:
       - AWS_SECRET_ACCESS_KEY
       - AWS_SESSION_TOKEN
     env_file:
-      - .env
+      - path: .env
+        required: false
     ports:
       - "4040:4040"
       - "18080:18080"