From be1ff4778bdee0f8545dfab1f01a0ec67972e29d Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 16 Jul 2024 09:52:26 -0700 Subject: [PATCH] Adding the automated test step in the docker build push job itself. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using “test before push” workflow mentioned in docker build push action official documentation. Link: https://docs.docker.com/build/ci/github-actions/test-before-push/ --- While this would work, a problem is that I need to put the docker compose step in the docker build push job. This means, it would run for all matrix jobs. In our case currently manager service image takes time to build, and manager depends on mqtt service image which is built pretty quickly. So we are good here. But consider this scenario. What if a matrix job B is dependent on matrix job A but matrix job A hasn’t finished building it’s image as yet, then matrix job B might use the older image which it would have pulled and not the locally built / updated image. In this case, the tests would be run on an older image, and could falsely pass the tests. --- For instance, in this test workflow run, I wanted to test the success / failure test case and used a different test script. This is copied over in the manager/Dockerfile. But since the automated tests step is part of the same matrix job, it would run for all services and not just manager which has the entry point (run-test.sh) script for automated tests. So, mqtt service image, in this case, also executed the docker compose command, it wasn’t able to find the test script in the manager image. The reason is that while I did include the test script in the manager/Dockerfile, this image is still being built, hence latest changes not yet available. So, the matrix job for the mqtt service uses the pulled image layers from the cache for the manager image (it pulled in the first place since the manager image was used as a part of the docker compose), and this pulled image did not have the test script file. Hence the workflow failed saying, no such file found: https://github.com/MukuFlash03/everest-demo/actions/runs/9960438276/job/27519676597 ---- One workaround I see then is to run the automated tests conditionally only for the manager service matrix job. --- .github/workflows/cicd.yaml | 64 +++++++++++++----------------- docker-compose.automated-tests.yml | 4 -- 2 files changed, 28 insertions(+), 40 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index ce55ef12..0a8e8c83 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -2,47 +2,16 @@ name: cicd on: pull_request: - branches: [ main ] + branches: + - main push: - branches: [ main ] + branches: + - main tags: - 'v[0-9]+.[0-9]+.[0-9]+' jobs: - run-demo-automated-tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: List Docker images - run: docker images - - - name: Run automated tests using docker-compose.automated-tests.yml - timeout-minutes: 30 - run: | - echo "Running docker compose up..." - docker compose --project-name everest-ac-demo \ - --file "docker-compose.automated-tests.yml" up \ - --build \ - --abort-on-container-exit \ - --exit-code-from manager - - exit_code=$? - echo "Docker-compose up exit code from manager service: $exit_code" - - echo "Running docker compose down..." - docker compose --project-name everest-ac-demo \ - --file "docker-compose.automated-tests.yml" down - - - name: List Docker images - run: docker images - docker-build-and-push-images: - needs: [run-demo-automated-tests] runs-on: ubuntu-latest permissions: @@ -125,8 +94,31 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and export to Docker + uses: docker/build-push-action@v6 + with: + load: true + context: ${{ matrix.context }} + push: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=${{ matrix.image_name }} + cache-to: type=gha,mode=max,scope=${{ matrix.image_name }} + + - name: Run automated tests using docker-compose.automated-tests.yml + if: ${{ matrix.image_name == 'manager' }} + run: | + echo "Running docker compose up..." + docker compose --project-name everest-ac-demo \ + --file "docker-compose.automated-tests.yml" up \ + --abort-on-container-exit \ + --exit-code-from manager + + exit_code=$? + echo "Docker-compose up exit code from manager service: $exit_code" + - name: Build and push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ${{ matrix.context }} push: ${{ github.event_name != 'pull_request' }} diff --git a/docker-compose.automated-tests.yml b/docker-compose.automated-tests.yml index aacfa9ff..0dd561e4 100644 --- a/docker-compose.automated-tests.yml +++ b/docker-compose.automated-tests.yml @@ -3,15 +3,11 @@ version: "3.6" services: mqtt-server: image: ghcr.io/everest/everest-demo/mqtt-server:${TAG} - build: mosquitto - pull_policy: build logging: driver: none manager: image: ghcr.io/everest/everest-demo/manager:${TAG} - build: manager - pull_policy: build depends_on: - mqtt-server environment: