Merge pull request #640 from DeepRank/529_add_docker_testing_action_g…

…croci2

ci: add action for releasing new docker image and testing it

.github/workflows/_ghcr.yml

@@ -0,0 +1,70 @@
+# SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all)
+# SPDX-FileCopyrightText: 2022 dv4all
+# SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) <e.cahen@esciencecenter.nl>
+# SPDX-FileCopyrightText: 2024 Netherlands eScience Center
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: reusable ghcr.io module
+
+on:
+  workflow_call:
+    inputs:
+      ghcr_user:
+        required: true
+        description: User for logging to ghcr.io (use github.actor)
+        type: string
+      base_image_name:
+        required: true
+        description: Base image name incl. ghcr.io
+        type: string
+      image_tag:
+        required: true
+        description: Image tag (version)
+        type: string
+      dockerfile:
+        required: true
+        description: Location and name of docker file
+        type: string
+      docker_context:
+        required: true
+        description: Docker context for the build command
+        type: string
+    secrets:
+      token:
+        required: true
+    outputs:
+      image_created:
+        description: Full image name after upload to ghcr.io
+        value: ${{jobs.build_and_push.outputs.image_build}}
+      image_uploaded:
+        description: Confirmation that image is uploaded to ghcr.io
+        value: ${{jobs.build_and_push.outputs.image_pushed}}
+
+jobs:
+  build_and_push:
+    name: build and push image
+    runs-on: ubuntu-22.04
+    outputs:
+      image_build: ${{steps.build_image.outputs.image_build}}
+      image_pushed: ${{steps.build_image.outputs.image_pushed}}
+    steps:
+      - name: checkout
+        # https://github.com/actions/checkout
+        uses: actions/checkout@v4
+      - name: build
+        id: build_image
+        run: |
+          IMAGE_TAG_VERSION=${{inputs.base_image_name}}:${{inputs.image_tag}}
+          IMAGE_TAG_LASTEST=${{inputs.base_image_name}}:latest
+          echo image_tag_version $IMAGE_TAG_VERSION
+          docker build -t $IMAGE_TAG_VERSION -t $IMAGE_TAG_LASTEST -f ${{inputs.dockerfile}} ${{inputs.docker_context}}
+          echo "{image_build}={$IMAGE_TAG_VERSIONvalue}" >> $GITHUB_OUTPUT
+      - name: push to ghcr.io
+        id: push_image
+        run: |
+          echo login
+          echo "${{secrets.token}}" | docker login https://ghcr.io -u ${{inputs.ghcr_user}} --password-stdin
+          echo push auth image with all tags
+          docker push ${{inputs.base_image_name}} --all-tags
+          echo "{image_build}={$IMAGE_TAG_VERSIONvalue}" >> $GITHUB_OUTPUT

.github/workflows/build-latest-release.yml

@@ -3,7 +3,7 @@ name: build (latest release)
 # Only trigger, when the release workflow succeeded
 on:
   workflow_run:
-    workflows: ["Build and upload to PyPI"]
+    workflows: ["Build and upload to PyPI and ghcr.io"]
     types:
       - completed
 
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest"]
+        os: ["ubuntu-22.04"]
         python-version: ["3.10"] # ["3.10", "3.11"]
 
     # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell

.github/workflows/build-repo-frozen-env.yml

@@ -36,7 +36,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest"]
+        os: ["ubuntu-22.04"]
         python-version: ["3.10"] # ["3.10", "3.11"]
 
     # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell

.github/workflows/build-repo.yml

@@ -36,7 +36,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest"]
+        os: ["ubuntu-22.04"]
         python-version: ["3.10"] # ["3.10", "3.11"]
 
     # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell

.github/workflows/cffconvert.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   validate:
     name: "validate"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Check out a copy of the repository
         uses: actions/checkout@v3

.github/workflows/coveralls.yml

@@ -36,7 +36,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest"]
+        os: ["ubuntu-22.04"]
         python-version: ["3.10"]
 
     # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell

.github/workflows/draft-pdf.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   paper:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Paper Draft
     steps:
       - name: Checkout

.github/workflows/fair-software.yml

@@ -11,7 +11,7 @@ jobs:
   verify:
     if: github.event.pull_request.draft == false
     name: "fair-software"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: fair-software/howfairis-github-action@0.2.1
         name: Measure compliance with fair-software.eu recommendations

.github/workflows/linting.yml

@@ -36,7 +36,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest"]
+        os: ["ubuntu-22.04"]
         python-version: ["3.10"]
 
     # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell

.github/workflows/markdown-link-check.yml

@@ -23,7 +23,7 @@ jobs:
   markdown-link-check:
     if: github.event.pull_request.draft == false
     name: Check markdown links
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
       - uses: gaurav-nelson/github-action-markdown-link-check@v1

.github/workflows/notebooks.yml

@@ -34,7 +34,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest"]
+        os: ["ubuntu-22.04"]
         python-version: ["3.10"] # ["3.10", "3.11"]
 
     # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell

.github/workflows/release_pypi.yml

@@ -1,4 +1,4 @@
-name: Build and upload to PyPI
+name: Build and upload to PyPI and ghcr.io
 
 on:
   workflow_dispatch:
@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest"]
+        os: ["ubuntu-22.04"]
         python-version: ["3.10"]
 
     # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell
@@ -36,8 +36,8 @@ jobs:
           path: dist/*
 
   upload_test_pypi:
-    needs: [build]
-    runs-on: ubuntu-latest
+    needs: build
+    runs-on: ubuntu-22.04
     if: github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/download-artifact@v3
@@ -51,8 +51,8 @@ jobs:
           repository_url: https://test.pypi.org/legacy/
 
   upload_pypi:
-    needs: [build]
-    runs-on: ubuntu-latest
+    needs: build
+    runs-on: ubuntu-22.04
     if: github.event_name == 'release' && github.event.action == 'published'
     steps:
       - uses: actions/download-artifact@v3
@@ -63,3 +63,39 @@ jobs:
         with:
           user: __token__
           password: ${{ secrets.PYPI_TOKEN_DEEPRANK2 }}
+
+  read_only_version:
+    needs: upload_pypi
+    name: Read version from TOML
+    runs-on: ubuntu-22.04
+    outputs:
+      version: ${{ steps.get_version.outputs.VERSION }}
+      repo_lowercase: ${{ steps.repo_lowercase.outputs.REPO_LOWERCASE }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Read version from TOML
+        id: get_version
+        run: |
+          VERSION=$(grep '^version =' pyproject.toml | awk -F '"' '{print $2}')
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Convert repository name to lowercase
+        id: repo_lowercase
+        run: |
+          REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
+          echo "REPO_LOWERCASE=$REPO_LOWERCASE" >> $GITHUB_OUTPUT
+
+  upload_docker_image:
+    needs: read_only_version
+    name: Upload Docker image to ghcr.io
+    uses: ./.github/workflows/_ghcr.yml
+    with:
+      ghcr_user: ${{github.actor}}
+      base_image_name: ghcr.io/${{ needs.read_only_version.outputs.repo_lowercase }}
+      image_tag: ${{ needs.read_only_version.outputs.version }}
+      dockerfile: ./Dockerfile
+      docker_context: .
+    secrets:
+      token: ${{secrets.GITHUB_TOKEN}}

.github/workflows/stale_issue_pr.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   close-issues:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       issues: write
       pull-requests: write

.github/workflows/test-docker-image.yml

@@ -0,0 +1,64 @@
+name: test latest docker image
+
+# Only trigger, when the release workflow succeeded
+on:
+  workflow_run:
+    workflows: ["Build and upload to PyPI and ghcr.io"]
+    types:
+      - completed
+
+jobs:
+  test_latest_docker_image:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull latest Docker image
+        run: |
+          REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
+          docker pull ghcr.io/$REPO_LOWERCASE:latest
+
+      - name: Run tests in Docker container
+        run: |
+
+          REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
+          docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null
+
+          PROJECT_DIR="/app"
+          TEST_DIR="$PROJECT_DIR/tests"
+          echo "Project directory: $PROJECT_DIR"
+          echo "Tests directory: $TEST_DIR"
+
+          # Create project directory and copy tests folder
+          docker exec test_container mkdir -p $PROJECT_DIR
+          docker cp tests test_container:$TEST_DIR
+
+          # Verify the directory structure
+          echo "Contents of project directory:"
+          docker exec test_container ls -la $PROJECT_DIR
+          echo "Contents of tests directory:"
+          docker exec test_container ls -la $TEST_DIR
+
+          # Install pytest
+          docker exec test_container pip install pytest
+
+          # Run pytest from the project directory
+          echo "Running pytest from the project directory:"
+          docker exec -w $PROJECT_DIR test_container python -m pytest tests -v
+
+          # Clean up
+          docker stop test_container
+          docker rm test_container
+
+      - name: Output test results
+        if: failure()
+        run: |
+          echo "Tests failed. Please check the test output above for more details."

Dockerfile

@@ -22,7 +22,8 @@ RUN \
   echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \
   echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc
 
-ADD ./env/deeprank2.yml /home/deeprank2
+ADD ./env/deeprank2.yml /home/deeprank2/
+ADD ./tutorials /home/deeprank2/tutorials
 
 RUN \
   ## Create the environment and install the dependencies
@@ -35,7 +36,7 @@ RUN \
   # Get the data for running the tutorials
   if [ -d "/home/deeprank2/tutorials/data_raw" ]; then rm -Rf /home/deeprank2/tutorials/data_raw; fi && \
   if [ -d "/home/deeprank2/tutorials/data_processed" ]; then rm -Rf /home/deeprank2/tutorials/data_processed; fi && \
-  wget https://zenodo.org/records/8349335/files/data_raw.zip && \
+  wget https://zenodo.org/records/13709906/files/data_raw.zip && \
   unzip data_raw.zip -d data_raw && \
   mv data_raw /home/deeprank2/tutorials && \
   apt-get clean && \
@@ -45,8 +46,6 @@ RUN \
   find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \
   conda clean --force-pkgs-dirs --all --yes
 
-ADD ./tutorials /home/deeprank2/tutorials
-
 ENV PATH /opt/conda/envs/deeprank2/bin:$PATH
 
 # Define working directory