[temp] Implement best practices (poetry, pre-commit, GH checks)

.dockerignore

@@ -158,3 +158,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+data/

.env.example

@@ -8,4 +8,4 @@ PATH_BASE=/data  # dir in container / within code
 PATH_HOST_BASE=/dir/on/host # dir on host, e.g. /home/riesgroup/temp/decode_cloud/mounts
 
 TIMEOUT_JOB=30
-TIMOUT_STATUS=30
+TIMEOUT_STATUS=30

.github/workflows/code-checks.yaml

@@ -0,0 +1,60 @@
+name: Code checks
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  static_code_checks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install poetry
+      run: pipx install poetry
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11.10'
+        cache: 'poetry'
+    - name: Install dependencies
+      run: poetry install
+    - name: Ruff check
+      run: poetry run ruff check .
+    - name: Ruff format check
+      run: poetry run ruff format --check .
+    - name: Mypy check
+      run: poetry run mypy .
+  tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install poetry
+      run: pipx install poetry
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11.10'
+        cache: 'poetry'
+    - name: Install dependencies
+      run: poetry install
+    - name: configure AWS
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ secrets.TESTS_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.TESTS_AWS_SECRET_ACCESS_KEY }}
+        aws-region: eu-central-1
+    - name: Run tests
+      run: |
+        poetry run pytest -m "aws or not(aws)" --junitxml=pytest.xml --cov-report=term-missing --cov=cli --cov=fetcher | tee pytest-coverage.txt
+        echo "test_exit_code=${PIPESTATUS[0]}" >> $GITHUB_ENV
+    - name: Coverage comment
+      uses: MishaKav/pytest-coverage-comment@main
+      with:
+        pytest-coverage-path: ./pytest-coverage.txt
+        junitxml-path: ./pytest.xml
+    - name: Fail on test failure
+      run: exit ${{ env.test_exit_code }}

.gitignore

@@ -158,3 +158,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+data/

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+  - repo: local
+    hooks:
+      - id: ruff-check
+        name: ruff check
+        entry: poetry run ruff check --fix .
+        language: system
+        types: [python]
+      - id: ruff-format
+        name: ruff format
+        entry: poetry run ruff format --check .
+        language: system
+        types: [python]
+
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: poetry run mypy
+        language: system
+        types: [python]

Dockerfile

@@ -1,11 +1,14 @@
-# see README.md for usage
-FROM python:3.10
+# Limitation:
+#   - Big image size: uses poetry, gets the whole repo, ...
+#   - No caching/multi-stage build: the whole image is rebuilt every time
+# Done this way for easier mapping AWS AppRunner from GitHub source <-> Dockerfile
 
-WORKDIR /app
+ARG PYTHON_VERSION
 
-COPY requirements.txt /app
-RUN pip install -r requirements.txt
+FROM python:${PYTHON_VERSION}-slim as builder
 
-COPY . /app
+WORKDIR /app
 
-CMD ["python", "-m", "cli.main"]
+COPY . /app/
+RUN chmod +x /app/scripts/setup.sh && /app/scripts/setup.sh
+CMD ["poetry", "run", "run"]

README.md

@@ -23,15 +23,15 @@ Copy the `.env.example` file to a `.env` file at the root of the directory and d
     - `PATH_HOST_BASE`: path to mount on the host (e.g., `/home/user/temp/decode_cloud/mount).
   - Timeouts:
     - `TIMEOUT_JOB`: how often (in seconds) to look for a new job.
-    - `TIMOUT_STATUS`: how often (in seconds) to send a keep-alive signal while processing the job.
+    - `TIMEOUT_STATUS`: how often (in seconds) to send a keep-alive signal while processing the job.
 #### Build the Docker image
 `docker build -t jobfetcher .`
 #### Install the nvidia-container-toolkit
 [See here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) to run GPU jobs
 #### Run the Docker container
 `docker run --env-file .env --gpus '"device=0"' -v <PATH_HOST_BASE>:<PATH_BASE> -v /var/run/docker.sock:/var/run/docker.sock --add-host=host.docker.internal:host-gateway jobfetcher:latest`, where:
  - `<PATH_HOST_BASE>` and `<PATH_BASE>` are set as above.
- - `--add-host=host.docker.internal:host-gateway jobfetcher:latest` is required only when running Linux.
+ - `--add-host=host.docker.internal:host-gateway` is required only when running Linux.
  - The `--gpus '"device=0"'` option specifies which GPUs the worker should be able to use. `--gpus all` selects all GPUs, but you typically want to select which GPU to reserve, and if you have many, run multiple workers each with one reserved GPU.
 
 ## Test locally