From a31f88811e7fca572bffca162407ad1aafd61a72 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Tue, 18 Jul 2023 18:13:44 +0200 Subject: [PATCH 1/9] Update README mqa folder - Fix temporal elements - Fix templates --- log/mqa/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log/mqa/README.md b/log/mqa/README.md index 5ccada8..63d5e85 100644 --- a/log/mqa/README.md +++ b/log/mqa/README.md @@ -1,2 +1,2 @@ # Log folder -This folder is used to store logs generated by `ckan2pycsw` script and `catalog.rdf` file downloaded from URL. Logs are essential for debugging and tracing the execution of the code. By keeping logs, you can track the behavior of your code over time, identify errors and performance issues, and improve the overall quality of your code. \ No newline at end of file +This folder is used to store logs generated by `ckan2pycsw` script and `catalog.rdf` file downloaded from `CKAN_CATALOG_URL`. Logs are essential for debugging and tracing the execution of the code. By keeping logs, you can track the behavior of your code over time, identify errors and performance issues, and improve the overall quality of your code. \ No newline at end of file From 939144d5ba505dbb091537f504910c738b649cb0 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Fri, 2 Feb 2024 04:06:15 +0000 Subject: [PATCH 2/9] fix: Dockerfile.dev to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-DEBIAN12-GLIBC-6210088 - https://snyk.io/vuln/SNYK-DEBIAN12-GLIBC-6210088 - https://snyk.io/vuln/SNYK-DEBIAN12-GLIBC-6210098 - https://snyk.io/vuln/SNYK-DEBIAN12-GLIBC-6210099 - https://snyk.io/vuln/SNYK-DEBIAN12-ZLIB-6008963 --- Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index c206a4e..1fce3f1 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.13.0a3-slim LABEL maintainer="mnl.janez@gmail.com" ENV APP_DIR=/app From ab16c5e901d4f826fb42434897d4fc9d66b630f1 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 8 Feb 2024 10:01:31 +0100 Subject: [PATCH 3/9] Improve workflows (Test, lint & scan) New docker-pr.yml, docker-manual.yml and docker-push.yml. #2 --- .github/drafts/docker.yaml | 51 +++++++++++ .github/workflows/docker-manual.yaml | 70 +++++++++++++++ .github/workflows/docker-pr.yaml | 72 +++++++++++++++ .github/workflows/docker.yaml | 128 ++++++++++++++++----------- 4 files changed, 270 insertions(+), 51 deletions(-) create mode 100644 .github/drafts/docker.yaml create mode 100644 .github/workflows/docker-manual.yaml create mode 100644 .github/workflows/docker-pr.yaml diff --git a/.github/drafts/docker.yaml b/.github/drafts/docker.yaml new file mode 100644 index 0000000..b0d3177 --- /dev/null +++ b/.github/drafts/docker.yaml @@ -0,0 +1,51 @@ +name: Docker + +on: + push: + branches: + - main + - latest + tags: + - 'v*.*.*' + pull_request: + branches: + - main + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Checkout + uses: actions/checkout@v3 + - + name: Login to registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - + name: Build and push + uses: docker/build-push-action@v3 + with: + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docker-manual.yaml b/.github/workflows/docker-manual.yaml new file mode 100644 index 0000000..c792661 --- /dev/null +++ b/.github/workflows/docker-manual.yaml @@ -0,0 +1,70 @@ +name: Build and release image manually + +on: workflow_dispatch + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + TAG: ghcr.io/${{ github.repository }}:${{ github.ref_name }} + CONTEXT: . + BRANCH: ${{ github.ref_name }} + DOCKERFILE_PATH: . + DOCKERFILE: Dockerfile + +jobs: + docker: + name: manual/build-docker-push:${{ github.ref_name }} + runs-on: ubuntu-latest + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Check out code + uses: actions/checkout@v4 + + - name: Login to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + labels: | + org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md + org.opencontainers.image.version=${{ env.BRANCH }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + push: true + tags: ${{ env.TAG }} + labels: ${{ steps.meta.outputs.labels }} + context: ${{ env.CONTEXT }} + file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + + - name: Linting Dockerfile with hadolint in GH Actions + uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + + - name: Run Trivy container image vulnerability scanner + uses: aquasecurity/trivy-action@0.17.0 + with: + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + format: sarif + output: trivy-results.sarif + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + if: always() + with: + sarif_file: trivy-results.sarif \ No newline at end of file diff --git a/.github/workflows/docker-pr.yaml b/.github/workflows/docker-pr.yaml new file mode 100644 index 0000000..9ce3a72 --- /dev/null +++ b/.github/workflows/docker-pr.yaml @@ -0,0 +1,72 @@ +name: Test ckan-mqa images (PR) + +on: + pull_request: + branches: + - main + - latest + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + CONTEXT: . + BRANCH: ${{ github.head_ref }} + DOCKERFILE_PATH: . + DOCKERFILE: Dockerfile + HADOLINT_VERSION: 2.12.0 + +jobs: + docker: + name: runner/test-docker-pr:${{ github.head_ref }} + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Checkout + uses: actions/checkout@v4 + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + labels: | + org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md + org.opencontainers.image.version=${{ env.BRANCH }} + + - name: Build to test + uses: docker/build-push-action@v5 + id: docker-push + with: + push: false + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + labels: ${{ steps.meta.outputs.labels }} + context: ${{ env.CONTEXT }} + file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + + - name: Linting Dockerfile and annotate code inline in the github PR viewer + id: hadolint + uses: jbergstroem/hadolint-gh-action@v1.12.2 + with: + dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + version: ${{ env.HADOLINT_VERSION }} + annotate: true + error_level: -1 + + - name: Run Trivy container image vulnerability scanner + uses: aquasecurity/trivy-action@0.17.0 + with: + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + format: sarif + output: trivy-results.sarif + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + if: always() + with: + sarif_file: trivy-results.sarif \ No newline at end of file diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index b0d3177..c054b7b 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -1,51 +1,77 @@ -name: Docker - -on: - push: - branches: - - main - - latest - tags: - - 'v*.*.*' - pull_request: - branches: - - main - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Checkout - uses: actions/checkout@v3 - - - name: Login to registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Build and push - uses: docker/build-push-action@v3 - with: - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} +name: Build and push ckan-mqa image from PR Merge + +on: + pull_request: + types: + - closed + branches: + - main + - latest + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + TAG: ghcr.io/${{ github.repository }}:${{ github.head_ref }} + CONTEXT: . + BRANCH: ${{ github.head_ref }} + DOCKERFILE_PATH: . + DOCKERFILE: Dockerfile + +jobs: + docker: + name: runner/build-docker-push:${{ github.head_ref }} + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Check out code + uses: actions/checkout@v4 + + - name: Login to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + labels: | + org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md + org.opencontainers.image.version=${{ env.BRANCH }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + push: true + tags: ${{ env.TAG }} + labels: ${{ steps.meta.outputs.labels }} + context: ${{ env.CONTEXT }} + file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + + - name: Linting Dockerfile with hadolint in GH Actions + uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + + - name: Run Trivy container image vulnerability scanner + uses: aquasecurity/trivy-action@0.17.0 + with: + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + format: sarif + output: trivy-results.sarif + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + if: always() + with: + sarif_file: trivy-results.sarif \ No newline at end of file From c5b2200626ab92f839d688659103c3036a819ae7 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 8 Feb 2024 10:34:13 +0100 Subject: [PATCH 4/9] Fix tag names with slashes --- .github/workflows/docker-manual.yaml | 6 ++++-- .github/workflows/docker-pr.yaml | 5 ++++- .github/workflows/docker.yaml | 6 ++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-manual.yaml b/.github/workflows/docker-manual.yaml index c792661..7a70c5b 100644 --- a/.github/workflows/docker-manual.yaml +++ b/.github/workflows/docker-manual.yaml @@ -5,7 +5,6 @@ on: workflow_dispatch env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - TAG: ghcr.io/${{ github.repository }}:${{ github.ref_name }} CONTEXT: . BRANCH: ${{ github.ref_name }} DOCKERFILE_PATH: . @@ -42,11 +41,14 @@ jobs: org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md org.opencontainers.image.version=${{ env.BRANCH }} + - name: Replace slashes in BRANCH to avoid errors + run: echo "BRANCH=${BRANCH////_}" >> $GITHUB_ENV + - name: Build and push uses: docker/build-push-action@v5 with: push: true - tags: ${{ env.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} labels: ${{ steps.meta.outputs.labels }} context: ${{ env.CONTEXT }} file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} diff --git a/.github/workflows/docker-pr.yaml b/.github/workflows/docker-pr.yaml index 9ce3a72..0edda11 100644 --- a/.github/workflows/docker-pr.yaml +++ b/.github/workflows/docker-pr.yaml @@ -1,4 +1,4 @@ -name: Test ckan-mqa images (PR) +name: Test ckan-pycsw images (PR) on: pull_request: @@ -39,6 +39,9 @@ jobs: org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md org.opencontainers.image.version=${{ env.BRANCH }} + - name: Replace slashes in BRANCH to avoid errors + run: echo "BRANCH=${BRANCH////_}" >> $GITHUB_ENV + - name: Build to test uses: docker/build-push-action@v5 id: docker-push diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index c054b7b..8a2fca4 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -11,7 +11,6 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - TAG: ghcr.io/${{ github.repository }}:${{ github.head_ref }} CONTEXT: . BRANCH: ${{ github.head_ref }} DOCKERFILE_PATH: . @@ -49,11 +48,14 @@ jobs: org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md org.opencontainers.image.version=${{ env.BRANCH }} + - name: Replace slashes in BRANCH to avoid errors + run: echo "BRANCH=${BRANCH////_}" >> $GITHUB_ENV + - name: Build and push uses: docker/build-push-action@v5 with: push: true - tags: ${{ env.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} labels: ${{ steps.meta.outputs.labels }} context: ${{ env.CONTEXT }} file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} From 50620a702d44c3a35cfb6279d6720d8a2967b409 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 8 Feb 2024 10:56:19 +0100 Subject: [PATCH 5/9] Fix DOCKERFILE_PATH --- .github/workflows/docker-manual.yaml | 2 +- .github/workflows/docker-pr.yaml | 4 ++-- .github/workflows/docker.yaml | 2 +- Dockerfile => ckan-mqa/Dockerfile | 2 +- Dockerfile.dev => ckan-mqa/Dockerfile.dev | 4 ++-- ckan-mqa/{ => docker-entrypoint.d}/entrypoint.sh | 0 ckan-mqa/{ => docker-entrypoint.d}/entrypoint_dev.sh | 0 docker-compose.dev.yml | 2 +- docker-compose.yml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) rename Dockerfile => ckan-mqa/Dockerfile (91%) rename Dockerfile.dev => ckan-mqa/Dockerfile.dev (84%) rename ckan-mqa/{ => docker-entrypoint.d}/entrypoint.sh (100%) rename ckan-mqa/{ => docker-entrypoint.d}/entrypoint_dev.sh (100%) diff --git a/.github/workflows/docker-manual.yaml b/.github/workflows/docker-manual.yaml index 7a70c5b..9d13b9c 100644 --- a/.github/workflows/docker-manual.yaml +++ b/.github/workflows/docker-manual.yaml @@ -7,7 +7,7 @@ env: IMAGE_NAME: ${{ github.repository }} CONTEXT: . BRANCH: ${{ github.ref_name }} - DOCKERFILE_PATH: . + DOCKERFILE_PATH: /ckan-mqa DOCKERFILE: Dockerfile jobs: diff --git a/.github/workflows/docker-pr.yaml b/.github/workflows/docker-pr.yaml index 0edda11..7a22f77 100644 --- a/.github/workflows/docker-pr.yaml +++ b/.github/workflows/docker-pr.yaml @@ -1,4 +1,4 @@ -name: Test ckan-pycsw images (PR) +name: Test ckan-mqa images (PR) on: pull_request: @@ -11,7 +11,7 @@ env: IMAGE_NAME: ${{ github.repository }} CONTEXT: . BRANCH: ${{ github.head_ref }} - DOCKERFILE_PATH: . + DOCKERFILE_PATH: /ckan-mqa DOCKERFILE: Dockerfile HADOLINT_VERSION: 2.12.0 diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 8a2fca4..5b56f25 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -13,7 +13,7 @@ env: IMAGE_NAME: ${{ github.repository }} CONTEXT: . BRANCH: ${{ github.head_ref }} - DOCKERFILE_PATH: . + DOCKERFILE_PATH: /ckan-mqa DOCKERFILE: Dockerfile jobs: diff --git a/Dockerfile b/ckan-mqa/Dockerfile similarity index 91% rename from Dockerfile rename to ckan-mqa/Dockerfile index f24f3a5..fbdd8e8 100644 --- a/Dockerfile +++ b/ckan-mqa/Dockerfile @@ -21,7 +21,7 @@ COPY pyproject.toml pdm.lock . RUN pdm install --no-self --group prod -COPY ckan-mqa/entrypoint.sh entrypoint.sh +COPY ckan-mqa/docker-entrypoint.d/entrypoint.sh entrypoint.sh COPY ckan2mqa ckan2mqa ENTRYPOINT ["/bin/bash", "./entrypoint.sh"] \ No newline at end of file diff --git a/Dockerfile.dev b/ckan-mqa/Dockerfile.dev similarity index 84% rename from Dockerfile.dev rename to ckan-mqa/Dockerfile.dev index 1fce3f1..ac442f2 100644 --- a/Dockerfile.dev +++ b/ckan-mqa/Dockerfile.dev @@ -21,11 +21,11 @@ COPY pyproject.toml pdm.lock . RUN pdm install --no-self --group prod -COPY ckan-mqa/entrypoint_dev.sh entrypoint.sh +COPY ckan-mqa/docker-entrypoint.d/entrypoint_dev.sh entrypoint.sh COPY ckan2mqa ckan2mqa EXPOSE 5678/TCP # Set entrypoint with debugpy -ENTRYPOINT ["python3", "-m", "debugpy", "--listen", "0.0.0.0:5678", "--wait-for-client", "./entrypoint_dev.sh"] +ENTRYPOINT ["python3", "-m", "debugpy", "--listen", "0.0.0.0:5678", "--wait-for-client", "./ckan-mqa/docker-entrypoint.d/entrypoint_dev.sh"] CMD ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/ckan-mqa/entrypoint.sh b/ckan-mqa/docker-entrypoint.d/entrypoint.sh similarity index 100% rename from ckan-mqa/entrypoint.sh rename to ckan-mqa/docker-entrypoint.d/entrypoint.sh diff --git a/ckan-mqa/entrypoint_dev.sh b/ckan-mqa/docker-entrypoint.d/entrypoint_dev.sh similarity index 100% rename from ckan-mqa/entrypoint_dev.sh rename to ckan-mqa/docker-entrypoint.d/entrypoint_dev.sh diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e9735b1..d3b149e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -4,7 +4,7 @@ services: container_name: ckan-mqa-dev build: context: . - dockerfile: Dockerfile.dev + dockerfile: ckan-mqa/Dockerfile.dev env_file: - .env logging: diff --git a/docker-compose.yml b/docker-compose.yml index 7e39db7..c9a6937 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: container_name: ckan-mqa build: context: . - dockerfile: Dockerfile + dockerfile: ckan-mqa/Dockerfile image: ghcr.io/mjanez/ckan-mqa:latest env_file: - .env From 9c35ebf4c3dc37f6d646ed22585ce4e280a89a10 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 8 Feb 2024 10:59:21 +0100 Subject: [PATCH 6/9] Fix Dockerfile lint error COPY with more than 2 arguments requires the last argument to end with / https://github.com/mjanez/ckan-mqa/pull/5/files#annotation_17839586951 --- ckan-mqa/Dockerfile | 2 +- ckan-mqa/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan-mqa/Dockerfile b/ckan-mqa/Dockerfile index fbdd8e8..7af96e9 100644 --- a/ckan-mqa/Dockerfile +++ b/ckan-mqa/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get -q -y update && \ python3 -m pip install pdm WORKDIR ${APP_DIR} -COPY pyproject.toml pdm.lock . +COPY pyproject.toml pdm.lock ./ RUN pdm install --no-self --group prod diff --git a/ckan-mqa/Dockerfile.dev b/ckan-mqa/Dockerfile.dev index ac442f2..7d2cbc0 100644 --- a/ckan-mqa/Dockerfile.dev +++ b/ckan-mqa/Dockerfile.dev @@ -17,7 +17,7 @@ RUN apt-get -q -y update && \ python3 -m pip install pdm debugpy WORKDIR ${APP_DIR} -COPY pyproject.toml pdm.lock . +COPY pyproject.toml pdm.lock ./ RUN pdm install --no-self --group prod From f1b1463caedfc896b6d27b686660e1c1a32b00e4 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 8 Feb 2024 11:14:46 +0100 Subject: [PATCH 7/9] Improve docker actions --- .github/workflows/docker-manual.yaml | 3 +- .github/workflows/docker-pr.yaml | 46 ++++++---------------------- .github/workflows/docker.yaml | 16 ++++++---- 3 files changed, 22 insertions(+), 43 deletions(-) diff --git a/.github/workflows/docker-manual.yaml b/.github/workflows/docker-manual.yaml index 9d13b9c..a8da4d7 100644 --- a/.github/workflows/docker-manual.yaml +++ b/.github/workflows/docker-manual.yaml @@ -50,13 +50,14 @@ jobs: push: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} labels: ${{ steps.meta.outputs.labels }} - context: ${{ env.CONTEXT }} + context: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }} file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} - name: Linting Dockerfile with hadolint in GH Actions uses: hadolint/hadolint-action@v3.1.0 with: dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + no-fail: true - name: Run Trivy container image vulnerability scanner uses: aquasecurity/trivy-action@0.17.0 diff --git a/.github/workflows/docker-pr.yaml b/.github/workflows/docker-pr.yaml index 7a22f77..4071b4e 100644 --- a/.github/workflows/docker-pr.yaml +++ b/.github/workflows/docker-pr.yaml @@ -3,12 +3,15 @@ name: Test ckan-mqa images (PR) on: pull_request: branches: - - main - - latest + - master + - 'ckan-mqa-*.*.*' + - '!dev/ckan-mqa-*.*.*' + - '!feature/*' + - '!fix/*' env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} + IMAGE_NAME: ckan-mqa CONTEXT: . BRANCH: ${{ github.head_ref }} DOCKERFILE_PATH: /ckan-mqa @@ -17,7 +20,7 @@ env: jobs: docker: - name: runner/test-docker-pr:${{ github.head_ref }} + name: runner/test-ckan-mqa-pr:${{ github.head_ref }} runs-on: ubuntu-latest if: github.event_name == 'pull_request' steps: @@ -35,41 +38,12 @@ jobs: uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - labels: | - org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md - org.opencontainers.image.version=${{ env.BRANCH }} - - name: Replace slashes in BRANCH to avoid errors - run: echo "BRANCH=${BRANCH////_}" >> $GITHUB_ENV - - - name: Build to test - uses: docker/build-push-action@v5 - id: docker-push - with: - push: false - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} - labels: ${{ steps.meta.outputs.labels }} - context: ${{ env.CONTEXT }} - file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} - - - name: Linting Dockerfile and annotate code inline in the github PR viewer + - name: Linting Dockerfiles and annotate code inline in the github PR viewer id: hadolint uses: jbergstroem/hadolint-gh-action@v1.12.2 with: - dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}.ghcr version: ${{ env.HADOLINT_VERSION }} annotate: true - error_level: -1 - - - name: Run Trivy container image vulnerability scanner - uses: aquasecurity/trivy-action@0.17.0 - with: - image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} - format: sarif - output: trivy-results.sarif - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 - if: always() - with: - sarif_file: trivy-results.sarif \ No newline at end of file + error_level: -1 \ No newline at end of file diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 5b56f25..bde6a66 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -3,10 +3,13 @@ name: Build and push ckan-mqa image from PR Merge on: pull_request: types: - - closed + - closed branches: - - main - - latest + - master + - 'ckan-mqa-*.*.*' + - '!dev/ckan-mqa-*.*.*' + - '!feature/*' + - '!fix/*' env: REGISTRY: ghcr.io @@ -55,18 +58,19 @@ jobs: uses: docker/build-push-action@v5 with: push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + tags: ghcr.io/${{ github.repository }}:${{ github.head_ref }} labels: ${{ steps.meta.outputs.labels }} - context: ${{ env.CONTEXT }} + context: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }} file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} - name: Linting Dockerfile with hadolint in GH Actions uses: hadolint/hadolint-action@v3.1.0 with: dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + no-fail: true - name: Run Trivy container image vulnerability scanner - uses: aquasecurity/trivy-action@0.17.0 + uses: aquasecurity/trivy-action@0.12.0 with: image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} format: sarif From e2d71e1a9d53d482f4cfbf48e488d95322417b70 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 8 Feb 2024 12:09:49 +0100 Subject: [PATCH 8/9] Fix ckan2mqa --- ckan-mqa/Dockerfile | 2 +- ckan-mqa/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan-mqa/Dockerfile b/ckan-mqa/Dockerfile index 7af96e9..6095d7c 100644 --- a/ckan-mqa/Dockerfile +++ b/ckan-mqa/Dockerfile @@ -22,6 +22,6 @@ COPY pyproject.toml pdm.lock ./ RUN pdm install --no-self --group prod COPY ckan-mqa/docker-entrypoint.d/entrypoint.sh entrypoint.sh -COPY ckan2mqa ckan2mqa +COPY ./ckan2mqa /ckan2mqa ENTRYPOINT ["/bin/bash", "./entrypoint.sh"] \ No newline at end of file diff --git a/ckan-mqa/Dockerfile.dev b/ckan-mqa/Dockerfile.dev index 7d2cbc0..ac4dec0 100644 --- a/ckan-mqa/Dockerfile.dev +++ b/ckan-mqa/Dockerfile.dev @@ -22,7 +22,7 @@ COPY pyproject.toml pdm.lock ./ RUN pdm install --no-self --group prod COPY ckan-mqa/docker-entrypoint.d/entrypoint_dev.sh entrypoint.sh -COPY ckan2mqa ckan2mqa +COPY ./ckan2mqa /ckan2mqa EXPOSE 5678/TCP From 4e604c889c8fd62be43351dc8e8fcd32e622affa Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 8 Feb 2024 12:21:11 +0100 Subject: [PATCH 9/9] Fix context --- .github/drafts/docker.yaml | 2 +- .github/workflows/docker-manual.yaml | 6 +++--- .github/workflows/docker-pr.yaml | 4 ++-- .github/workflows/docker.yaml | 6 +++--- ckan-mqa/Dockerfile | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/drafts/docker.yaml b/.github/drafts/docker.yaml index b0d3177..2bbb993 100644 --- a/.github/drafts/docker.yaml +++ b/.github/drafts/docker.yaml @@ -39,7 +39,7 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - diff --git a/.github/workflows/docker-manual.yaml b/.github/workflows/docker-manual.yaml index a8da4d7..b594ca4 100644 --- a/.github/workflows/docker-manual.yaml +++ b/.github/workflows/docker-manual.yaml @@ -34,7 +34,7 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} labels: | @@ -50,7 +50,7 @@ jobs: push: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} labels: ${{ steps.meta.outputs.labels }} - context: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }} + context: ${{ env.CONTEXT }} file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} - name: Linting Dockerfile with hadolint in GH Actions @@ -67,7 +67,7 @@ jobs: output: trivy-results.sarif - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 if: always() with: sarif_file: trivy-results.sarif \ No newline at end of file diff --git a/.github/workflows/docker-pr.yaml b/.github/workflows/docker-pr.yaml index 4071b4e..a1e147e 100644 --- a/.github/workflows/docker-pr.yaml +++ b/.github/workflows/docker-pr.yaml @@ -35,7 +35,7 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} @@ -43,7 +43,7 @@ jobs: id: hadolint uses: jbergstroem/hadolint-gh-action@v1.12.2 with: - dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}.ghcr + dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} version: ${{ env.HADOLINT_VERSION }} annotate: true error_level: -1 \ No newline at end of file diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index bde6a66..8e5c167 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -44,7 +44,7 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} labels: | @@ -60,7 +60,7 @@ jobs: push: true tags: ghcr.io/${{ github.repository }}:${{ github.head_ref }} labels: ${{ steps.meta.outputs.labels }} - context: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }} + context: ${{ env.CONTEXT }} file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} - name: Linting Dockerfile with hadolint in GH Actions @@ -77,7 +77,7 @@ jobs: output: trivy-results.sarif - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 if: always() with: sarif_file: trivy-results.sarif \ No newline at end of file diff --git a/ckan-mqa/Dockerfile b/ckan-mqa/Dockerfile index 6095d7c..183a03a 100644 --- a/ckan-mqa/Dockerfile +++ b/ckan-mqa/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.13.0a3-slim LABEL maintainer="mnl.janez@gmail.com" ENV APP_DIR=/app