From 3ea7fed112001d22f1f818ba9d0f1a9c424bf25d Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 29 Jan 2025 07:27:42 +0000 Subject: [PATCH 01/91] ci: move to GHA; only run ghcr build on test pass AND when tagged --- .circleci/config.yml | 11 ----- .github/workflows/ghcr.yml | 10 ++-- .github/workflows/test-nginx.yml | 28 ----------- .github/workflows/test.yml | 47 +++++++++++++++++++ docker-compose.yml | 12 ++--- test/{ => nginx}/.gitignore | 0 .../files/nginx-test/acme-challenge | 0 .../files/nginx-test/http_root/index.html | 0 .../nginx-test/http_root/should-be-cached.txt | 0 .../files/nginx-test/http_root/version.txt | 0 test/{ => nginx}/mock-http-server/.gitignore | 0 test/{ => nginx}/mock-http-server/index.js | 0 .../mock-http-server/package-lock.json | 0 .../{ => nginx}/mock-http-server/package.json | 0 test/{ => nginx}/mock-http-service.dockerfile | 0 .../{ => nginx}/nginx.test.docker-compose.yml | 0 test/{ => nginx}/package-lock.json | 0 test/{ => nginx}/package.json | 0 test/{ => nginx}/run-tests.sh | 0 test/{ => nginx}/test-nginx.js | 0 test/test-images.sh | 35 ++++++++++++++ 21 files changed, 95 insertions(+), 48 deletions(-) delete mode 100644 .github/workflows/test-nginx.yml create mode 100644 .github/workflows/test.yml rename test/{ => nginx}/.gitignore (100%) rename test/{ => nginx}/files/nginx-test/acme-challenge (100%) rename test/{ => nginx}/files/nginx-test/http_root/index.html (100%) rename test/{ => nginx}/files/nginx-test/http_root/should-be-cached.txt (100%) rename test/{ => nginx}/files/nginx-test/http_root/version.txt (100%) rename test/{ => nginx}/mock-http-server/.gitignore (100%) rename test/{ => nginx}/mock-http-server/index.js (100%) rename test/{ => nginx}/mock-http-server/package-lock.json (100%) rename test/{ => nginx}/mock-http-server/package.json (100%) rename test/{ => nginx}/mock-http-service.dockerfile (100%) rename test/{ => nginx}/nginx.test.docker-compose.yml (100%) rename test/{ => nginx}/package-lock.json (100%) rename test/{ => nginx}/package.json (100%) rename test/{ => nginx}/run-tests.sh (100%) rename test/{ => nginx}/test-nginx.js (100%) create mode 100755 test/test-images.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 01960b5e..3266b6e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,17 +1,6 @@ version: 2 jobs: build: - machine: - image: ubuntu-2004:current - - steps: - - checkout - - - run: sudo apt install shellcheck - - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | xargs shellcheck --exclude=SC2016 - - - run: git submodule update -i - - run: | echo 'SSL_TYPE=selfsign DOMAIN=local diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index c7659113..2243fb59 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -1,16 +1,20 @@ name: GHCR on: - workflow_dispatch: - push: - branches: [master] + # See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_run + workflow_run: + workflows: [Test] + types: [completed] tags: ["v*.*.*"] + workflow_dispatch: env: REGISTRY: ghcr.io jobs: build-push-image: + # Only run this job if tests pass. See workflow_run definition above. + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/test-nginx.yml b/.github/workflows/test-nginx.yml deleted file mode 100644 index cd1349c8..00000000 --- a/.github/workflows/test-nginx.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Test nginx config - -on: - push: - pull_request: - -jobs: - build: - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - submodules: recursive - - uses: actions/setup-node@v4 - with: - node-version: 22.12.0 - - run: cd test && npm i - - run: cd test && ./run-tests.sh - - - if: always() - run: docker logs test-nginx-1 || true - - if: always() - run: docker logs test-service-1 || true - - if: always() - run: docker logs test-enketo-1 || true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..12acb04b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,47 @@ +name: Test + +on: + push: + pull_request: + +jobs: + shellcheck: + timeout-minutes: 2 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: sudo apt-get install shellcheck + - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | xargs shellcheck --exclude=SC2016 + test-nginx: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + - uses: actions/setup-node@v4 + with: + node-version: 22.12.0 + - run: cd test/nginx && npm i + - run: cd test/nginx && ./run-tests.sh + + - if: always() + run: docker logs test-nginx-1 || true + - if: always() + run: docker logs test-service-1 || true + - if: always() + run: docker logs test-enketo-1 || true + test-images: + needs: + - shellcheck + - test-nginx + runs-on: ubuntu:latest # TODO matrix to run on all expected versions? + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + - run: ./test/test-images.sh diff --git a/docker-compose.yml b/docker-compose.yml index 97b7ac65..14d27495 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: postgres14: - build: + build: # TODO use explicit images - move building to test docker-compose file context: . dockerfile: postgres14.dockerfile volumes: @@ -13,7 +13,7 @@ services: postgres: # This service upgrades from postgres 9.6 to 14. # The legacy name must be maintained to allow access to the anonymous volume. - build: + build: # TODO use explicit images - move building to test docker-compose file context: . dockerfile: postgres-upgrade.dockerfile volumes: @@ -34,7 +34,7 @@ services: - DKIM_KEY_PATH=/etc/exim4/dkim.key.temp restart: always service: - build: + build: # TODO use explicit images - move building to test docker-compose file context: . dockerfile: service.dockerfile depends_on: @@ -79,7 +79,7 @@ services: logging: driver: local nginx: - build: + build: # TODO use explicit images - move building to test docker-compose file context: . dockerfile: nginx.dockerfile depends_on: @@ -113,14 +113,14 @@ services: secrets: volumes: - secrets:/etc/secrets - build: + build: # TODO use explicit images - move building to test docker-compose file context: . dockerfile: secrets.dockerfile command: './generate-secrets.sh' enketo: volumes: - secrets:/etc/secrets - build: + build: # TODO use explicit images - move building to test docker-compose file context: . dockerfile: enketo.dockerfile restart: always diff --git a/test/.gitignore b/test/nginx/.gitignore similarity index 100% rename from test/.gitignore rename to test/nginx/.gitignore diff --git a/test/files/nginx-test/acme-challenge b/test/nginx/files/nginx-test/acme-challenge similarity index 100% rename from test/files/nginx-test/acme-challenge rename to test/nginx/files/nginx-test/acme-challenge diff --git a/test/files/nginx-test/http_root/index.html b/test/nginx/files/nginx-test/http_root/index.html similarity index 100% rename from test/files/nginx-test/http_root/index.html rename to test/nginx/files/nginx-test/http_root/index.html diff --git a/test/files/nginx-test/http_root/should-be-cached.txt b/test/nginx/files/nginx-test/http_root/should-be-cached.txt similarity index 100% rename from test/files/nginx-test/http_root/should-be-cached.txt rename to test/nginx/files/nginx-test/http_root/should-be-cached.txt diff --git a/test/files/nginx-test/http_root/version.txt b/test/nginx/files/nginx-test/http_root/version.txt similarity index 100% rename from test/files/nginx-test/http_root/version.txt rename to test/nginx/files/nginx-test/http_root/version.txt diff --git a/test/mock-http-server/.gitignore b/test/nginx/mock-http-server/.gitignore similarity index 100% rename from test/mock-http-server/.gitignore rename to test/nginx/mock-http-server/.gitignore diff --git a/test/mock-http-server/index.js b/test/nginx/mock-http-server/index.js similarity index 100% rename from test/mock-http-server/index.js rename to test/nginx/mock-http-server/index.js diff --git a/test/mock-http-server/package-lock.json b/test/nginx/mock-http-server/package-lock.json similarity index 100% rename from test/mock-http-server/package-lock.json rename to test/nginx/mock-http-server/package-lock.json diff --git a/test/mock-http-server/package.json b/test/nginx/mock-http-server/package.json similarity index 100% rename from test/mock-http-server/package.json rename to test/nginx/mock-http-server/package.json diff --git a/test/mock-http-service.dockerfile b/test/nginx/mock-http-service.dockerfile similarity index 100% rename from test/mock-http-service.dockerfile rename to test/nginx/mock-http-service.dockerfile diff --git a/test/nginx.test.docker-compose.yml b/test/nginx/nginx.test.docker-compose.yml similarity index 100% rename from test/nginx.test.docker-compose.yml rename to test/nginx/nginx.test.docker-compose.yml diff --git a/test/package-lock.json b/test/nginx/package-lock.json similarity index 100% rename from test/package-lock.json rename to test/nginx/package-lock.json diff --git a/test/package.json b/test/nginx/package.json similarity index 100% rename from test/package.json rename to test/nginx/package.json diff --git a/test/run-tests.sh b/test/nginx/run-tests.sh similarity index 100% rename from test/run-tests.sh rename to test/nginx/run-tests.sh diff --git a/test/test-nginx.js b/test/nginx/test-nginx.js similarity index 100% rename from test/test-nginx.js rename to test/nginx/test-nginx.js diff --git a/test/test-images.sh b/test/test-images.sh new file mode 100755 index 00000000..ad6cb9c5 --- /dev/null +++ b/test/test-images.sh @@ -0,0 +1,35 @@ +#!/bin/bash -eux +set -o pipefail + +log() { echo >&2 "[$(basename "$0")] $*"; } + +echo 'SSL_TYPE=selfsign +DOMAIN=local +SYSADMIN_EMAIL=no-reply@getodk.org' > .env + +touch ./files/allow-postgres14-upgrade + +docker compose build + +# we allow a long retry period for the first check because the first-run +# nginx setup could take several minutes due to key generation. +log "Verifying frontend and backend load..." +docker compose up -d +CONTAINER_NAME=$(docker inspect -f '{{.Name}}' $(docker compose ps -q nginx) | cut -c2-) +docker run --network container:$CONTAINER_NAME \ + appropriate/curl -4 --insecure --retry 30 --retry-delay 10 --retry-connrefused https://localhost/ -H 'Host: local' \ +| tee /dev/tty \ +| grep -q 'ODK Central' + +docker run --network container:$CONTAINER_NAME \ + appropriate/curl -4 --insecure --retry 20 --retry-delay 2 --retry-connrefused https://localhost/v1/projects -H 'Host: local' \ +| tee /dev/tty \ +| grep -q '\[\]' + +log "Verifying pm2..." +docker compose exec -T service npx pm2 list \ +| tee /dev/tty \ +| grep -c "online" +| grep -q 4 + +log "All OK." From fc12898b48225af871c0a22dea85d25724f9a8ef Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 29 Jan 2025 07:32:16 +0000 Subject: [PATCH 02/91] fix shellcheck? --- test/test-images.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index ad6cb9c5..66e08b83 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -15,13 +15,13 @@ docker compose build # nginx setup could take several minutes due to key generation. log "Verifying frontend and backend load..." docker compose up -d -CONTAINER_NAME=$(docker inspect -f '{{.Name}}' $(docker compose ps -q nginx) | cut -c2-) -docker run --network container:$CONTAINER_NAME \ +CONTAINER_NAME=$(docker inspect -f '{{.Name}}' "$(docker compose ps -q nginx)" | cut -c2-) +docker run --network container:"$CONTAINER_NAME" \ appropriate/curl -4 --insecure --retry 30 --retry-delay 10 --retry-connrefused https://localhost/ -H 'Host: local' \ | tee /dev/tty \ | grep -q 'ODK Central' -docker run --network container:$CONTAINER_NAME \ +docker run --network container:"$CONTAINER_NAME" \ appropriate/curl -4 --insecure --retry 20 --retry-delay 2 --retry-connrefused https://localhost/v1/projects -H 'Host: local' \ | tee /dev/tty \ | grep -q '\[\]' @@ -29,7 +29,7 @@ docker run --network container:$CONTAINER_NAME \ log "Verifying pm2..." docker compose exec -T service npx pm2 list \ | tee /dev/tty \ -| grep -c "online" +| grep -c "online" \ | grep -q 4 log "All OK." From 4b71f945e507000fcac56ed6c440ac26642ee666 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 29 Jan 2025 07:33:03 +0000 Subject: [PATCH 03/91] fix docker-compose build context --- test/nginx/nginx.test.docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nginx/nginx.test.docker-compose.yml b/test/nginx/nginx.test.docker-compose.yml index 4a863054..d1d8e5e3 100644 --- a/test/nginx/nginx.test.docker-compose.yml +++ b/test/nginx/nginx.test.docker-compose.yml @@ -15,7 +15,7 @@ services: - PORT=8383 nginx: build: - context: .. + context: ../.. dockerfile: nginx.dockerfile args: SKIP_FRONTEND_BUILD: true From 1a628d79d4ee19751fe402e5e95acf70f88494f7 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 29 Jan 2025 07:41:01 +0000 Subject: [PATCH 04/91] volume paths --- test/nginx/nginx.test.docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/nginx/nginx.test.docker-compose.yml b/test/nginx/nginx.test.docker-compose.yml index d1d8e5e3..f930a0a8 100644 --- a/test/nginx/nginx.test.docker-compose.yml +++ b/test/nginx/nginx.test.docker-compose.yml @@ -28,8 +28,8 @@ services: - SSL_TYPE=selfsign - OIDC_ENABLED=false volumes: - - ../files/nginx/odk.conf.template:/usr/share/odk/nginx/odk.conf.template:ro - - ../files/nginx/client-config.json.template:/usr/share/odk/nginx/client-config.json.template:ro + - ../../files/nginx/odk.conf.template:/usr/share/odk/nginx/odk.conf.template:ro + - ../../files/nginx/client-config.json.template:/usr/share/odk/nginx/client-config.json.template:ro ports: - "9000:80" - "9001:443" From 46d2e175621811b1ead99171f900116d031b84ce Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 29 Jan 2025 07:56:02 +0000 Subject: [PATCH 05/91] ci: comment out working stuff --- .github/workflows/test.yml | 60 +++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 12acb04b..1cb112a0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,38 +5,38 @@ on: pull_request: jobs: - shellcheck: - timeout-minutes: 2 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: sudo apt-get install shellcheck - - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | xargs shellcheck --exclude=SC2016 - test-nginx: - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - submodules: recursive - - uses: actions/setup-node@v4 - with: - node-version: 22.12.0 - - run: cd test/nginx && npm i - - run: cd test/nginx && ./run-tests.sh + #shellcheck: + # timeout-minutes: 2 + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - run: sudo apt-get install shellcheck + # - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | xargs shellcheck --exclude=SC2016 + #test-nginx: + # timeout-minutes: 10 + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # submodules: recursive + # - uses: actions/setup-node@v4 + # with: + # node-version: 22.12.0 + # - run: cd test/nginx && npm i + # - run: cd test/nginx && ./run-tests.sh - - if: always() - run: docker logs test-nginx-1 || true - - if: always() - run: docker logs test-service-1 || true - - if: always() - run: docker logs test-enketo-1 || true + # - if: always() + # run: docker logs test-nginx-1 || true + # - if: always() + # run: docker logs test-service-1 || true + # - if: always() + # run: docker logs test-enketo-1 || true test-images: - needs: - - shellcheck - - test-nginx + #needs: + #- shellcheck + #- test-nginx runs-on: ubuntu:latest # TODO matrix to run on all expected versions? steps: - uses: actions/checkout@v4 From cde1280f4fd44f18217428964dab1db6a9d32822 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 29 Jan 2025 08:57:01 +0000 Subject: [PATCH 06/91] service.dockerfile: separate unrelated commands --- service.dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/service.dockerfile b/service.dockerfile index 7b947419..05b188bc 100644 --- a/service.dockerfile +++ b/service.dockerfile @@ -53,9 +53,9 @@ RUN apt-get update \ procps \ postgresql-client-14 \ netcat-traditional \ - && rm -rf /var/lib/apt/lists/* \ - && npm clean-install --omit=dev --no-audit \ - --fund=false --update-notifier=false + && rm -rf /var/lib/apt/lists/* +RUN npm clean-install --omit=dev --no-audit \ + --fund=false --update-notifier=false COPY server/ ./ COPY files/service/scripts/ ./ From f9bb335f7875fc8811b77752bbc5c8a04bbacdc8 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 29 Jan 2025 09:19:28 +0000 Subject: [PATCH 07/91] fix runner name --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1cb112a0..e6a673b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: #needs: #- shellcheck #- test-nginx - runs-on: ubuntu:latest # TODO matrix to run on all expected versions? + runs-on: ubuntu-latest # TODO matrix to run on all expected versions? steps: - uses: actions/checkout@v4 with: From fb8e4094de667e97d78a3d281ac11539a8c6f689 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 29 Jan 2025 11:16:54 +0000 Subject: [PATCH 08/91] check path with available tools --- test/test-images.sh | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 66e08b83..f29fc68b 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -3,6 +3,24 @@ set -o pipefail log() { echo >&2 "[$(basename "$0")] $*"; } +check_path() { + local timeout="$1" + local requestPath="$2" + local expected="$3" + + for i in 1.."$timeout"; do + if echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | + docker run -i container:"$CONTAINER_NAME" \ + openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 | + grep -q "$expected"; then + return + fi + done + + log "!!! Path $requestPath returned a non-OK HTTP status code: $status" + exit 1 +} + echo 'SSL_TYPE=selfsign DOMAIN=local SYSADMIN_EMAIL=no-reply@getodk.org' > .env @@ -15,16 +33,8 @@ docker compose build # nginx setup could take several minutes due to key generation. log "Verifying frontend and backend load..." docker compose up -d -CONTAINER_NAME=$(docker inspect -f '{{.Name}}' "$(docker compose ps -q nginx)" | cut -c2-) -docker run --network container:"$CONTAINER_NAME" \ - appropriate/curl -4 --insecure --retry 30 --retry-delay 10 --retry-connrefused https://localhost/ -H 'Host: local' \ -| tee /dev/tty \ -| grep -q 'ODK Central' - -docker run --network container:"$CONTAINER_NAME" \ - appropriate/curl -4 --insecure --retry 20 --retry-delay 2 --retry-connrefused https://localhost/v1/projects -H 'Host: local' \ -| tee /dev/tty \ -| grep -q '\[\]' +check_path 30 / 'ODK Central' +check_path 20 /v1/projects '[]' log "Verifying pm2..." docker compose exec -T service npx pm2 list \ From 185061562206805da2c9244ffae1d1179c56dd12 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:27:33 +0000 Subject: [PATCH 09/91] 1..timeout --- test/test-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index f29fc68b..cab2e897 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -8,7 +8,7 @@ check_path() { local requestPath="$2" local expected="$3" - for i in 1.."$timeout"; do + for i in {1.."$timeout"}; do if echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | docker run -i container:"$CONTAINER_NAME" \ openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 | From ae7d2c9caa9741a08baa3bb8b745f62002768557 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:29:34 +0000 Subject: [PATCH 10/91] for inc loop --- test/test-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index cab2e897..3361b966 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -8,7 +8,7 @@ check_path() { local requestPath="$2" local expected="$3" - for i in {1.."$timeout"}; do + for (( i=0; i<"$timeout"; ++i )); do if echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | docker run -i container:"$CONTAINER_NAME" \ openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 | From 6cf7ab950fd1cc69eaf239d123f420156cfd9c84 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:33:38 +0000 Subject: [PATCH 11/91] store result before testing --- test/test-images.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 3361b966..7b17251c 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -9,15 +9,18 @@ check_path() { local expected="$3" for (( i=0; i<"$timeout"; ++i )); do - if echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | + res = echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | docker run -i container:"$CONTAINER_NAME" \ - openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 | - grep -q "$expected"; then + openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 + if echo "$res" | grep -q "$expected"; then return fi done - log "!!! Path $requestPath returned a non-OK HTTP status code: $status" + log "!!! Path $requestPath returned unexpected result:" + echo + echo "$res" + echo exit 1 } From 12845ffee10abc7a585f00dd41c004cded21cde1 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:34:30 +0000 Subject: [PATCH 12/91] less subprocess --- test/test-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index 7b17251c..3fe8521b 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -12,7 +12,7 @@ check_path() { res = echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | docker run -i container:"$CONTAINER_NAME" \ openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 - if echo "$res" | grep -q "$expected"; then + if grep -q "$expected" <<<"$res"; then return fi done From 67b46a3bcb94ad4f14e59fe324fb2c8102b7e5c1 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:35:57 +0000 Subject: [PATCH 13/91] store res correct --- test/test-images.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 3fe8521b..dff11bd7 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -9,9 +9,11 @@ check_path() { local expected="$3" for (( i=0; i<"$timeout"; ++i )); do - res = echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | - docker run -i container:"$CONTAINER_NAME" \ - openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 + res="$( + echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | + docker run -i container:"$CONTAINER_NAME" \ + openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 + )" if grep -q "$expected" <<<"$res"; then return fi From d93db3d40402264d2f953ee76ad1817913f798c6 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:36:56 +0000 Subject: [PATCH 14/91] remove circle config --- .circleci/config.yml | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 3266b6e2..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: 2 -jobs: - build: - - run: | - echo 'SSL_TYPE=selfsign - DOMAIN=local - SYSADMIN_EMAIL=no-reply@getodk.org' > .env - - - run: touch ./files/allow-postgres14-upgrade - - - run: docker compose build - - - run: - # we allow a long retry period for the first check because the first-run - # nginx setup could take several minutes due to key generation. - name: Verify frontend and backend load - command: | - set -x - docker compose up -d - CONTAINER_NAME=$(docker inspect -f '{{.Name}}' $(docker compose ps -q nginx) | cut -c2-) - docker run --network container:$CONTAINER_NAME \ - appropriate/curl -4 --insecure --retry 30 --retry-delay 10 --retry-connrefused https://localhost/ -H 'Host: local' \ - | tee /dev/tty \ - | grep -q 'ODK Central' - docker run --network container:$CONTAINER_NAME \ - appropriate/curl -4 --insecure --retry 20 --retry-delay 2 --retry-connrefused https://localhost/v1/projects -H 'Host: local' \ - | tee /dev/tty \ - | grep -q '\[\]' - - run: - name: Verify pm2 - command: | - docker compose exec -T service npx pm2 list \ - | tee /dev/tty \ - | grep -c "online" | grep -q 4 From cbadcded2e13f30d08f770f2eb2a1e0d4287b418 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:37:27 +0000 Subject: [PATCH 15/91] get container name --- test/test-images.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test-images.sh b/test/test-images.sh index dff11bd7..61c47c56 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -33,6 +33,7 @@ SYSADMIN_EMAIL=no-reply@getodk.org' > .env touch ./files/allow-postgres14-upgrade docker compose build +CONTAINER_NAME="$(docker inspect -f '{{.Name}}' "$(docker compose ps -q nginx)" | cut -c2-)" # we allow a long retry period for the first check because the first-run # nginx setup could take several minutes due to key generation. From 4021da7b82c1f4d6683e9228e7d2859d88dacb4c Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:38:00 +0000 Subject: [PATCH 16/91] fix syntax hilighting --- test/test-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index 61c47c56..190c75b5 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -14,7 +14,7 @@ check_path() { docker run -i container:"$CONTAINER_NAME" \ openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 )" - if grep -q "$expected" <<<"$res"; then + if echo "$res" | grep -q "$expected"; then return fi done From 255cffc98487b11a4e9c24e8116e2aa1ce15590a Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:41:00 +0000 Subject: [PATCH 17/91] no container: prefix --- test/test-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index 190c75b5..7542562d 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -11,7 +11,7 @@ check_path() { for (( i=0; i<"$timeout"; ++i )); do res="$( echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | - docker run -i container:"$CONTAINER_NAME" \ + docker run -i "$CONTAINER_NAME" \ openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 )" if echo "$res" | grep -q "$expected"; then From 0c775f01ad6b47f5cd72aa02e1acdd816705ad19 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:43:14 +0000 Subject: [PATCH 18/91] start container before getting name --- test/test-images.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index 7542562d..0236c424 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -33,12 +33,16 @@ SYSADMIN_EMAIL=no-reply@getodk.org' > .env touch ./files/allow-postgres14-upgrade docker compose build + +log "Starting containers..." +docker compose up -d + +log "Getting nginx container ID..." CONTAINER_NAME="$(docker inspect -f '{{.Name}}' "$(docker compose ps -q nginx)" | cut -c2-)" # we allow a long retry period for the first check because the first-run # nginx setup could take several minutes due to key generation. log "Verifying frontend and backend load..." -docker compose up -d check_path 30 / 'ODK Central' check_path 20 /v1/projects '[]' From 69e78f31c929aaf1823c8c5144cfc76d5a586fd3 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 07:43:37 +0000 Subject: [PATCH 19/91] rename var --- test/test-images.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 0236c424..41544e1b 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -11,7 +11,7 @@ check_path() { for (( i=0; i<"$timeout"; ++i )); do res="$( echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | - docker run -i "$CONTAINER_NAME" \ + docker run -i "$nginxContainer" \ openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 )" if echo "$res" | grep -q "$expected"; then @@ -38,7 +38,7 @@ log "Starting containers..." docker compose up -d log "Getting nginx container ID..." -CONTAINER_NAME="$(docker inspect -f '{{.Name}}' "$(docker compose ps -q nginx)" | cut -c2-)" +nginxContainer="$(docker inspect -f '{{.Name}}' "$(docker compose ps -q nginx)" | cut -c2-)" # we allow a long retry period for the first check because the first-run # nginx setup could take several minutes due to key generation. From 3e32f1d9a1eb9e7ea12ae437660869421f46f1e0 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:02:25 +0000 Subject: [PATCH 20/91] exec --- test/test-images.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 41544e1b..0ca7a644 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -11,7 +11,7 @@ check_path() { for (( i=0; i<"$timeout"; ++i )); do res="$( echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | - docker run -i "$nginxContainer" \ + docker exec -i "$nginxContainer" \ openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 )" if echo "$res" | grep -q "$expected"; then @@ -38,7 +38,7 @@ log "Starting containers..." docker compose up -d log "Getting nginx container ID..." -nginxContainer="$(docker inspect -f '{{.Name}}' "$(docker compose ps -q nginx)" | cut -c2-)" +nginxContainer="$(docker compose ps -q nginx)" # we allow a long retry period for the first check because the first-run # nginx setup could take several minutes due to key generation. From f65e7f4a5d92812fd1aa5c6e9143d3089a73c4df Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:07:08 +0000 Subject: [PATCH 21/91] add logging --- test/test-images.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test-images.sh b/test/test-images.sh index 0ca7a644..e6977e7d 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -9,14 +9,18 @@ check_path() { local expected="$3" for (( i=0; i<"$timeout"; ++i )); do + log "Checking response from $requestPath..." res="$( echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | docker exec -i "$nginxContainer" \ openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 )" if echo "$res" | grep -q "$expected"; then + log " Request responded correctly." return fi + log " Request did not respond correctly." + sleep 1 done log "!!! Path $requestPath returned unexpected result:" From 798dddc3af647f946d8eb6b6668c172695e528d1 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:07:29 +0000 Subject: [PATCH 22/91] remove noisy x --- test/test-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index e6977e7d..3c97eb9b 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -1,4 +1,4 @@ -#!/bin/bash -eux +#!/bin/bash -eu set -o pipefail log() { echo >&2 "[$(basename "$0")] $*"; } From 6a213fe103566227091d9e8dc8a249afc8d3aa56 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:13:21 +0000 Subject: [PATCH 23/91] restructure --- test/test-images.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 3c97eb9b..d64b458d 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -4,11 +4,11 @@ set -o pipefail log() { echo >&2 "[$(basename "$0")] $*"; } check_path() { - local timeout="$1" + local retries="$1" local requestPath="$2" local expected="$3" - for (( i=0; i<"$timeout"; ++i )); do + for (( i=0; ; ++i )); do log "Checking response from $requestPath..." res="$( echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | @@ -19,15 +19,22 @@ check_path() { log " Request responded correctly." return fi + log " Request did not respond correctly." + + if [[ "$i" -ge "$retries" ]]; then + log "!!! Retry count exceeded." + log "!!! Final response:" + echo + echo "$res" + echo + + exit 1 + fi + + log " Sleeping..." sleep 1 done - - log "!!! Path $requestPath returned unexpected result:" - echo - echo "$res" - echo - exit 1 } echo 'SSL_TYPE=selfsign From 5cb22353d5ba71d43b86211a855fd71fb35d1f70 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:18:31 +0000 Subject: [PATCH 24/91] update check regex --- test/test-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index d64b458d..45efdd56 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -55,7 +55,7 @@ nginxContainer="$(docker compose ps -q nginx)" # nginx setup could take several minutes due to key generation. log "Verifying frontend and backend load..." check_path 30 / 'ODK Central' -check_path 20 /v1/projects '[]' +check_path 20 /v1/projects '\[\]' log "Verifying pm2..." docker compose exec -T service npx pm2 list \ From 11cb7a82fd69aced2d300652f004df96aff31bb2 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:34:49 +0000 Subject: [PATCH 25/91] clearer code --- test/test-images.sh | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 45efdd56..0469e87f 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -12,10 +12,11 @@ check_path() { log "Checking response from $requestPath..." res="$( echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | - docker exec -i "$nginxContainer" \ - openssl 2>&1 s_client -quiet -connect 127.0.0.1:443 + docker compose exec --no-TTY nginx \ + openssl s_client -quiet -connect 127.0.0.1:443 \ + 2>&1 )" - if echo "$res" | grep -q "$expected"; then + if echo "$res" | grep --silent --fixed-strings "$expected"; then log " Request responded correctly." return fi @@ -55,12 +56,21 @@ nginxContainer="$(docker compose ps -q nginx)" # nginx setup could take several minutes due to key generation. log "Verifying frontend and backend load..." check_path 30 / 'ODK Central' -check_path 20 /v1/projects '\[\]' +check_path 20 /v1/projects '[]' log "Verifying pm2..." -docker compose exec -T service npx pm2 list \ -| tee /dev/tty \ -| grep -c "online" \ -| grep -q 4 +processCount="$( + docker compose exec service npx pm2 list \ + | tee /dev/tty \ + | grep --count online +)" + +if [[ "$processCount" != 4 ]]; then + log "!!! PM2 returned an unexpected count for online processes." + log "!!!" + log "!!! Expected 4, but got $processCount." + + exit 1 +fi log "All OK." From ae5c3ea3be772da22c27281a8dd01a2d16180d45 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:35:16 +0000 Subject: [PATCH 26/91] remove nginxContainer var --- test/test-images.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 0469e87f..ef3f1128 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -49,9 +49,6 @@ docker compose build log "Starting containers..." docker compose up -d -log "Getting nginx container ID..." -nginxContainer="$(docker compose ps -q nginx)" - # we allow a long retry period for the first check because the first-run # nginx setup could take several minutes due to key generation. log "Verifying frontend and backend load..." From cbc2970f73618a5c66c5ca61afc14a64b84b37ef Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:41:40 +0000 Subject: [PATCH 27/91] test images --- test/test-images.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index ef3f1128..5f70e06a 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -44,10 +44,11 @@ SYSADMIN_EMAIL=no-reply@getodk.org' > .env touch ./files/allow-postgres14-upgrade +log "Building docker containers..." docker compose build log "Starting containers..." -docker compose up -d +docker compose up --detach # we allow a long retry period for the first check because the first-run # nginx setup could take several minutes due to key generation. From 213d2f99ec9c646f8bf7f14dcacfd5a19dd34e8b Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:44:32 +0000 Subject: [PATCH 28/91] add docker compose logs --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e6a673b3..05e24cca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,3 +45,5 @@ jobs: fetch-tags: true submodules: recursive - run: ./test/test-images.sh + - if: always() + run: docker compose logs From d868b9214b913637756e1d1b8c32ed673d25e6a8 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:45:54 +0000 Subject: [PATCH 29/91] X --- test/test-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index 5f70e06a..472191d7 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -1,4 +1,4 @@ -#!/bin/bash -eu +#!/bin/bash -eux set -o pipefail log() { echo >&2 "[$(basename "$0")] $*"; } From 0a006ae5dad1cae95a214fe1e880ba832857c510 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:51:00 +0000 Subject: [PATCH 30/91] or true? --- test/test-images.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 472191d7..2871191b 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -14,16 +14,17 @@ check_path() { echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | docker compose exec --no-TTY nginx \ openssl s_client -quiet -connect 127.0.0.1:443 \ - 2>&1 + 2>&1 || true )" if echo "$res" | grep --silent --fixed-strings "$expected"; then log " Request responded correctly." return fi - log " Request did not respond correctly." - - if [[ "$i" -ge "$retries" ]]; then + if [[ "$i" -lt "$retries" ]]; then + log " Request did not respond correctly; sleeping..." + sleep 1 + else log "!!! Retry count exceeded." log "!!! Final response:" echo @@ -32,9 +33,6 @@ check_path() { exit 1 fi - - log " Sleeping..." - sleep 1 done } From acc11474ed69556952ecec8aa94c35b4a7abfe21 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:56:27 +0000 Subject: [PATCH 31/91] use a temp file --- test/test-images.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 2871191b..0be750a4 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -1,8 +1,10 @@ -#!/bin/bash -eux +#!/bin/bash -eu set -o pipefail log() { echo >&2 "[$(basename "$0")] $*"; } +tmp="$(mktemp)" + check_path() { local retries="$1" local requestPath="$2" @@ -10,13 +12,12 @@ check_path() { for (( i=0; ; ++i )); do log "Checking response from $requestPath..." - res="$( - echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | - docker compose exec --no-TTY nginx \ - openssl s_client -quiet -connect 127.0.0.1:443 \ - 2>&1 || true - )" - if echo "$res" | grep --silent --fixed-strings "$expected"; then + echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | + docker compose exec --no-TTY nginx \ + openssl s_client -quiet -connect 127.0.0.1:443 \ + >"$tmp" 2>&1 \ + || true + if grep --silent --fixed-strings "$expected" "$tmp"; then log " Request responded correctly." return fi @@ -28,7 +29,7 @@ check_path() { log "!!! Retry count exceeded." log "!!! Final response:" echo - echo "$res" + cat "$tmp" echo exit 1 From 02fd5b140cea7420c8eaaef5c2ce63fb3682912e Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:57:19 +0000 Subject: [PATCH 32/91] single line --- test/test-images.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 0be750a4..7fecbf50 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -15,8 +15,7 @@ check_path() { echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | docker compose exec --no-TTY nginx \ openssl s_client -quiet -connect 127.0.0.1:443 \ - >"$tmp" 2>&1 \ - || true + >"$tmp" 2>&1 || true if grep --silent --fixed-strings "$expected" "$tmp"; then log " Request responded correctly." return @@ -44,10 +43,10 @@ SYSADMIN_EMAIL=no-reply@getodk.org' > .env touch ./files/allow-postgres14-upgrade log "Building docker containers..." -docker compose build +#docker compose build log "Starting containers..." -docker compose up --detach +#docker compose up --detach # we allow a long retry period for the first check because the first-run # nginx setup could take several minutes due to key generation. From 1f9a31e054d07be91a8ec1ea041aa3bd33b41a1b Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:58:54 +0000 Subject: [PATCH 33/91] redirect to stdout --- test/test-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index 7fecbf50..8bdba1f1 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -57,7 +57,7 @@ check_path 20 /v1/projects '[]' log "Verifying pm2..." processCount="$( docker compose exec service npx pm2 list \ - | tee /dev/tty \ + | tee /dev/stdout \ | grep --count online )" From f4b4456a0a085671d47f88dc6b458ee5d347b04f Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 08:59:56 +0000 Subject: [PATCH 34/91] run it --- test/test-images.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 8bdba1f1..87b959b0 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -43,10 +43,10 @@ SYSADMIN_EMAIL=no-reply@getodk.org' > .env touch ./files/allow-postgres14-upgrade log "Building docker containers..." -#docker compose build +docker compose build log "Starting containers..." -#docker compose up --detach +docker compose up --detach # we allow a long retry period for the first check because the first-run # nginx setup could take several minutes due to key generation. From f20b090f6e0cad990117180f5158e2aac16cd804 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:00:24 +0000 Subject: [PATCH 35/91] log spacing --- test/test-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index 87b959b0..28117a4b 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -11,7 +11,7 @@ check_path() { local expected="$3" for (( i=0; ; ++i )); do - log "Checking response from $requestPath..." + log "Checking response from $requestPath ..." echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | docker compose exec --no-TTY nginx \ openssl s_client -quiet -connect 127.0.0.1:443 \ From cdbefa065ab9bde4f921282d4c84bcef3af48e2c Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:04:45 +0000 Subject: [PATCH 36/91] show pm2 before count --- test/test-images.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 28117a4b..468d1876 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -15,7 +15,7 @@ check_path() { echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | docker compose exec --no-TTY nginx \ openssl s_client -quiet -connect 127.0.0.1:443 \ - >"$tmp" 2>&1 || true + >"$tmp" 2>&1 || true if grep --silent --fixed-strings "$expected" "$tmp"; then log " Request responded correctly." return @@ -55,12 +55,8 @@ check_path 30 / 'ODK Central' check_path 20 /v1/projects '[]' log "Verifying pm2..." -processCount="$( - docker compose exec service npx pm2 list \ - | tee /dev/stdout \ - | grep --count online -)" - +docker compose exec service npx pm2 list | tee "$tmp" +processCount="$(grep --count online "$tmp")" if [[ "$processCount" != 4 ]]; then log "!!! PM2 returned an unexpected count for online processes." log "!!!" From 4fe3ab6fe6e66db9f98067669263dea5daa873f2 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:05:04 +0000 Subject: [PATCH 37/91] 8 --- test/test-images.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 468d1876..5d36420d 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -57,10 +57,10 @@ check_path 20 /v1/projects '[]' log "Verifying pm2..." docker compose exec service npx pm2 list | tee "$tmp" processCount="$(grep --count online "$tmp")" -if [[ "$processCount" != 4 ]]; then +if [[ "$processCount" != 8 ]]; then log "!!! PM2 returned an unexpected count for online processes." log "!!!" - log "!!! Expected 4, but got $processCount." + log "!!! Expected 8, but got $processCount." exit 1 fi From 63faeefc237e8b173460a1a001dd57ad412bd51d Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:08:26 +0000 Subject: [PATCH 38/91] expect 4 --- test/test-images.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 5d36420d..468d1876 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -57,10 +57,10 @@ check_path 20 /v1/projects '[]' log "Verifying pm2..." docker compose exec service npx pm2 list | tee "$tmp" processCount="$(grep --count online "$tmp")" -if [[ "$processCount" != 8 ]]; then +if [[ "$processCount" != 4 ]]; then log "!!! PM2 returned an unexpected count for online processes." log "!!!" - log "!!! Expected 8, but got $processCount." + log "!!! Expected 4, but got $processCount." exit 1 fi From ef321d994a41fbda4abc9befc449a9d55dc22a98 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:10:01 +0000 Subject: [PATCH 39/91] enable all jobs --- .github/workflows/test.yml | 60 +++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05e24cca..b4b0b6e3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,38 +5,38 @@ on: pull_request: jobs: - #shellcheck: - # timeout-minutes: 2 - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # - run: sudo apt-get install shellcheck - # - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | xargs shellcheck --exclude=SC2016 - #test-nginx: - # timeout-minutes: 10 - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - # fetch-tags: true - # submodules: recursive - # - uses: actions/setup-node@v4 - # with: - # node-version: 22.12.0 - # - run: cd test/nginx && npm i - # - run: cd test/nginx && ./run-tests.sh + shellcheck: + timeout-minutes: 2 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: sudo apt-get install shellcheck + - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | xargs shellcheck --exclude=SC2016 + test-nginx: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + - uses: actions/setup-node@v4 + with: + node-version: 22.12.0 + - run: cd test/nginx && npm i + - run: cd test/nginx && ./run-tests.sh - # - if: always() - # run: docker logs test-nginx-1 || true - # - if: always() - # run: docker logs test-service-1 || true - # - if: always() - # run: docker logs test-enketo-1 || true + - if: always() + run: docker logs test-nginx-1 || true + - if: always() + run: docker logs test-service-1 || true + - if: always() + run: docker logs test-enketo-1 || true test-images: - #needs: - #- shellcheck - #- test-nginx + needs: + - shellcheck + - test-nginx runs-on: ubuntu-latest # TODO matrix to run on all expected versions? steps: - uses: actions/checkout@v4 From ba1f34d348ffad0df6557405a8c95ae66d5a8f24 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:13:16 +0000 Subject: [PATCH 40/91] clearer logging, comments --- test/test-images.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test-images.sh b/test/test-images.sh index 468d1876..69276066 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -48,11 +48,13 @@ docker compose build log "Starting containers..." docker compose up --detach -# we allow a long retry period for the first check because the first-run -# nginx setup could take several minutes due to key generation. -log "Verifying frontend and backend load..." +log "Verifying frontend..." check_path 30 / 'ODK Central' -check_path 20 /v1/projects '[]' +log " Frontend started OK." + +log "Verifying backend..." +check_path 2 /v1/projects '[]' +log " Backend started OK." log "Verifying pm2..." docker compose exec service npx pm2 list | tee "$tmp" From d09653ef3bda13d11ac9d89ffd3fb79bf20663d1 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:22:11 +0000 Subject: [PATCH 41/91] docker-compose: use published images --- docker-compose.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 14d27495..bb25eca0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,9 +34,7 @@ services: - DKIM_KEY_PATH=/etc/exim4/dkim.key.temp restart: always service: - build: # TODO use explicit images - move building to test docker-compose file - context: . - dockerfile: service.dockerfile + image: 'ghcr.io/getodk/central-service:2025.1' depends_on: - secrets - postgres14 @@ -79,9 +77,7 @@ services: logging: driver: local nginx: - build: # TODO use explicit images - move building to test docker-compose file - context: . - dockerfile: nginx.dockerfile + image: 'ghcr.io/getodk/central-nginx:2025.1' depends_on: - service - enketo From 9f190bd6e0c80374fd421c442fca05536a563553 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:23:32 +0000 Subject: [PATCH 42/91] use snapshots but not for root docker-compos --- test/snapshots.docker-compose.yml | 9 +++++++++ test/test-images.sh | 12 ++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 test/snapshots.docker-compose.yml diff --git a/test/snapshots.docker-compose.yml b/test/snapshots.docker-compose.yml new file mode 100644 index 00000000..21698ac8 --- /dev/null +++ b/test/snapshots.docker-compose.yml @@ -0,0 +1,9 @@ +services: + service: + build: + context: .. + dockerfile: service.dockerfile + nginx: + build: + context: .. + dockerfile: nginx.dockerfile diff --git a/test/test-images.sh b/test/test-images.sh index 69276066..84b12893 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -3,6 +3,10 @@ set -o pipefail log() { echo >&2 "[$(basename "$0")] $*"; } +docker_compose() { + docker compose --file test/snapshots.docker-compose.yml "$@" +} + tmp="$(mktemp)" check_path() { @@ -13,7 +17,7 @@ check_path() { for (( i=0; ; ++i )); do log "Checking response from $requestPath ..." echo -e "GET $requestPath HTTP/1.0\r\nHost: local\r\n\r\n" | - docker compose exec --no-TTY nginx \ + docker_compose exec --no-TTY nginx \ openssl s_client -quiet -connect 127.0.0.1:443 \ >"$tmp" 2>&1 || true if grep --silent --fixed-strings "$expected" "$tmp"; then @@ -43,10 +47,10 @@ SYSADMIN_EMAIL=no-reply@getodk.org' > .env touch ./files/allow-postgres14-upgrade log "Building docker containers..." -docker compose build +docker_compose build log "Starting containers..." -docker compose up --detach +docker_compose up --detach log "Verifying frontend..." check_path 30 / 'ODK Central' @@ -57,7 +61,7 @@ check_path 2 /v1/projects '[]' log " Backend started OK." log "Verifying pm2..." -docker compose exec service npx pm2 list | tee "$tmp" +docker_compose exec service npx pm2 list | tee "$tmp" processCount="$(grep --count online "$tmp")" if [[ "$processCount" != 4 ]]; then log "!!! PM2 returned an unexpected count for online processes." From 0dfed52e3be7b84210ec250233a823d5ff5d5fda Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:34:04 +0000 Subject: [PATCH 43/91] use both docker-compose files --- test/test-images.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test-images.sh b/test/test-images.sh index 84b12893..04e7f828 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -4,7 +4,10 @@ set -o pipefail log() { echo >&2 "[$(basename "$0")] $*"; } docker_compose() { - docker compose --file test/snapshots.docker-compose.yml "$@" + docker compose \ + --file test/snapshots.docker-compose.yml \ + --file docker-compose.yml \ + "$@" } tmp="$(mktemp)" From 9c85d21bf0fd684a16c54529ac00dbf66f049c8f Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:35:23 +0000 Subject: [PATCH 44/91] ci: add TODO --- .github/workflows/ghcr.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 2243fb59..304ca48a 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -32,6 +32,9 @@ jobs: fetch-tags: true submodules: recursive + - name: Confirm version numbers + run: echo TODO + - name: Log into registry ${{ env.REGISTRY }} uses: docker/login-action@v3 with: From 8a54f30df82bbb19973aee05b8841c1ac0177da1 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:43:26 +0000 Subject: [PATCH 45/91] fix compose file order --- docker-compose.yml | 8 ++++---- test/test-images.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bb25eca0..56513718 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: postgres14: - build: # TODO use explicit images - move building to test docker-compose file + build: context: . dockerfile: postgres14.dockerfile volumes: @@ -13,7 +13,7 @@ services: postgres: # This service upgrades from postgres 9.6 to 14. # The legacy name must be maintained to allow access to the anonymous volume. - build: # TODO use explicit images - move building to test docker-compose file + build: context: . dockerfile: postgres-upgrade.dockerfile volumes: @@ -109,14 +109,14 @@ services: secrets: volumes: - secrets:/etc/secrets - build: # TODO use explicit images - move building to test docker-compose file + build: context: . dockerfile: secrets.dockerfile command: './generate-secrets.sh' enketo: volumes: - secrets:/etc/secrets - build: # TODO use explicit images - move building to test docker-compose file + build: context: . dockerfile: enketo.dockerfile restart: always diff --git a/test/test-images.sh b/test/test-images.sh index 04e7f828..3cb97756 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -5,8 +5,8 @@ log() { echo >&2 "[$(basename "$0")] $*"; } docker_compose() { docker compose \ - --file test/snapshots.docker-compose.yml \ --file docker-compose.yml \ + --file test/snapshots.docker-compose.yml \ "$@" } From 12e7ab80bf391564863932fffc2311096b11b027 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:44:54 +0000 Subject: [PATCH 46/91] fix context --- test/snapshots.docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/snapshots.docker-compose.yml b/test/snapshots.docker-compose.yml index 21698ac8..5e276f85 100644 --- a/test/snapshots.docker-compose.yml +++ b/test/snapshots.docker-compose.yml @@ -1,9 +1,9 @@ services: service: build: - context: .. + context: . dockerfile: service.dockerfile nginx: build: - context: .. + context: . dockerfile: nginx.dockerfile From ba309a5fbaaab96970ce6f1ff28da036e86937b0 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:45:15 +0000 Subject: [PATCH 47/91] remove explicit context --- test/snapshots.docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/snapshots.docker-compose.yml b/test/snapshots.docker-compose.yml index 5e276f85..4635e85d 100644 --- a/test/snapshots.docker-compose.yml +++ b/test/snapshots.docker-compose.yml @@ -1,9 +1,7 @@ services: service: build: - context: . dockerfile: service.dockerfile nginx: build: - context: . dockerfile: nginx.dockerfile From e9c8398a177d7ff2828e04c9516737e69d2dc163 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:49:30 +0000 Subject: [PATCH 48/91] ghcr: show expected tag --- .github/workflows/ghcr.yml | 46 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 304ca48a..988d8b38 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -32,34 +32,34 @@ jobs: fetch-tags: true submodules: recursive - - name: Confirm version numbers - run: echo TODO - - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract Docker metadata id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - - name: Set up QEMU emulator for multi-arch images - uses: docker/setup-qemu-action@v3 + - name: Confirm version numbers + run: echo "TODO Expected version: ${{ steps.meta.outputs.tags }}" - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + #- name: Log into registry ${{ env.REGISTRY }} + # uses: docker/login-action@v3 + # with: + # registry: ${{ env.REGISTRY }} + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push ${{ matrix.image }} Docker image - uses: docker/build-push-action@v5 - with: - file: ${{ matrix.image }}.dockerfile - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: 'linux/amd64,linux/arm64' + #- name: Set up QEMU emulator for multi-arch images + # uses: docker/setup-qemu-action@v3 + + #- name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + #- name: Build and push ${{ matrix.image }} Docker image + # uses: docker/build-push-action@v5 + # with: + # file: ${{ matrix.image }}.dockerfile + # context: . + # push: true + # tags: ${{ steps.meta.outputs.tags }} + # labels: ${{ steps.meta.outputs.labels }} + # platforms: 'linux/amd64,linux/arm64' From 3bcb62cae4845783d94074cce31bb6e7ab6ed4a9 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:49:55 +0000 Subject: [PATCH 49/91] run ghcr job --- .github/workflows/ghcr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 988d8b38..d276237f 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -7,6 +7,7 @@ on: types: [completed] tags: ["v*.*.*"] workflow_dispatch: + pull_request: # disable later env: REGISTRY: ghcr.io From dbdb89339ed6d5ec2c7fc29f832079e89ad7c930 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 09:50:43 +0000 Subject: [PATCH 50/91] docker run --- .github/workflows/ghcr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index d276237f..14a2b6f5 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -40,7 +40,7 @@ jobs: images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - name: Confirm version numbers - run: echo "TODO Expected version: ${{ steps.meta.outputs.tags }}" + run: 'echo "TODO Expected version: ${{ steps.meta.outputs.tags }}"' #- name: Log into registry ${{ env.REGISTRY }} # uses: docker/login-action@v3 From fea213054240e45c88bea0f393e2da3d9508ceec Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:09:31 +0000 Subject: [PATCH 51/91] try execute? --- .github/workflows/ghcr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 14a2b6f5..90099ae3 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -7,7 +7,9 @@ on: types: [completed] tags: ["v*.*.*"] workflow_dispatch: - pull_request: # disable later + # disable later + push: + pull_request: env: REGISTRY: ghcr.io From 9241ef1623059e2158cc356197223bb706ed945d Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:11:39 +0000 Subject: [PATCH 52/91] try wider conditionsal --- .github/workflows/ghcr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 90099ae3..623561a0 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -7,7 +7,7 @@ on: types: [completed] tags: ["v*.*.*"] workflow_dispatch: - # disable later + # disable later: push: pull_request: @@ -17,7 +17,7 @@ env: jobs: build-push-image: # Only run this job if tests pass. See workflow_run definition above. - if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} runs-on: ubuntu-latest permissions: contents: read From bb5a23042a1a605776325560fb2eba983cb06e17 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:12:13 +0000 Subject: [PATCH 53/91] comment --- .github/workflows/ghcr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 623561a0..95d89f89 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -17,6 +17,7 @@ env: jobs: build-push-image: # Only run this job if tests pass. See workflow_run definition above. + # TODO remove unused part os this if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} runs-on: ubuntu-latest permissions: From 660d375bc3cdcbb3cebdb174991af00dcefb62db Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:12:40 +0000 Subject: [PATCH 54/91] comment out test yml --- .github/workflows/test.yml | 98 +++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b4b0b6e3..0681d764 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,49 +1,49 @@ -name: Test - -on: - push: - pull_request: - -jobs: - shellcheck: - timeout-minutes: 2 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: sudo apt-get install shellcheck - - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | xargs shellcheck --exclude=SC2016 - test-nginx: - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - submodules: recursive - - uses: actions/setup-node@v4 - with: - node-version: 22.12.0 - - run: cd test/nginx && npm i - - run: cd test/nginx && ./run-tests.sh - - - if: always() - run: docker logs test-nginx-1 || true - - if: always() - run: docker logs test-service-1 || true - - if: always() - run: docker logs test-enketo-1 || true - test-images: - needs: - - shellcheck - - test-nginx - runs-on: ubuntu-latest # TODO matrix to run on all expected versions? - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - submodules: recursive - - run: ./test/test-images.sh - - if: always() - run: docker compose logs +#name: Test +# +#on: +# push: +# pull_request: +# +#jobs: +# shellcheck: +# timeout-minutes: 2 +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - run: sudo apt-get install shellcheck +# - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | xargs shellcheck --exclude=SC2016 +# test-nginx: +# timeout-minutes: 10 +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# fetch-tags: true +# submodules: recursive +# - uses: actions/setup-node@v4 +# with: +# node-version: 22.12.0 +# - run: cd test/nginx && npm i +# - run: cd test/nginx && ./run-tests.sh +# +# - if: always() +# run: docker logs test-nginx-1 || true +# - if: always() +# run: docker logs test-service-1 || true +# - if: always() +# run: docker logs test-enketo-1 || true +# test-images: +# needs: +# - shellcheck +# - test-nginx +# runs-on: ubuntu-latest # TODO matrix to run on all expected versions? +# steps: +# - uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# fetch-tags: true +# submodules: recursive +# - run: ./test/test-images.sh +# - if: always() +# run: docker compose logs From 20580f9ac988df1981179127926a6b5662675e70 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:14:43 +0000 Subject: [PATCH 55/91] print event name --- .github/workflows/ghcr.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 95d89f89..ac63a469 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -42,8 +42,9 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - - name: Confirm version numbers - run: 'echo "TODO Expected version: ${{ steps.meta.outputs.tags }}"' + - run: 'echo "Confirm version numbers"' + run: 'echo "- steps.meta.outputs.tags: ${{ steps.meta.outputs.tags }}"' + run: 'echo "- github.event_name: ${{ github.event_name }}"' #- name: Log into registry ${{ env.REGISTRY }} # uses: docker/login-action@v3 From 86a263a56f9c4c1c5bed14e6fbc4ae9be9ef70a5 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:16:38 +0000 Subject: [PATCH 56/91] move test yml out the way --- .github/workflows/test.yml => test.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/test.yml => test.yml (100%) diff --git a/.github/workflows/test.yml b/test.yml similarity index 100% rename from .github/workflows/test.yml rename to test.yml From e6e20cc5041533e6d2f4c8ed6b28935a158ce14d Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:17:24 +0000 Subject: [PATCH 57/91] fix yml --- .github/workflows/ghcr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index ac63a469..8ae4d640 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -43,8 +43,8 @@ jobs: images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - run: 'echo "Confirm version numbers"' - run: 'echo "- steps.meta.outputs.tags: ${{ steps.meta.outputs.tags }}"' - run: 'echo "- github.event_name: ${{ github.event_name }}"' + - run: 'echo "- steps.meta.outputs.tags: ${{ steps.meta.outputs.tags }}"' + - run: 'echo "- github.event_name: ${{ github.event_name }}"' #- name: Log into registry ${{ env.REGISTRY }} # uses: docker/login-action@v3 From 7cb5681ffc9865641558ffcc1fb51ed9746e2abb Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:28:31 +0000 Subject: [PATCH 58/91] more devbug - tags etc --- .github/workflows/ghcr.yml | 45 ++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 8ae4d640..26f054be 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -14,11 +14,50 @@ on: env: REGISTRY: ghcr.io +# TODO move matrix up here? + jobs: + check-images-are-used: + # Only run this job if tests pass. See workflow_run definition above. + # TODO remove unused parts of this + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + needs: check-images-are-used + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + image: [nginx, service] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} + + - run: 'echo "TODO Confirm version numbers"' + - run: 'echo "- steps.meta.outputs.tags: ${{ steps.meta.outputs.tags }}"' + - run: 'echo "- github.event_name: ${{ github.event_name }}"' + - run: 'echo "- github.event_name: ${{ github.event_name }}"' + - run: 'echo "- GITHUB_REF: <$GITHUB_REF>"' + - run: 'echo "- GITHUB_REF_NAME: <$GITHUB_REF_NAME>"' + - run: 'echo "- GITHUB_REF_TYPE: <$GITHUB_REF_TYPE>"' + + - name: "Checking release image is referenced in docker-compose.yml..." + # TODO only run if tagged + run: 'grep "${{ steps.meta.outputs.tags }}" docker-compose.yml' + build-push-image: # Only run this job if tests pass. See workflow_run definition above. - # TODO remove unused part os this + # TODO remove unused parts of this if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + needs: check-images-are-used runs-on: ubuntu-latest permissions: contents: read @@ -42,9 +81,7 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - - run: 'echo "Confirm version numbers"' - - run: 'echo "- steps.meta.outputs.tags: ${{ steps.meta.outputs.tags }}"' - - run: 'echo "- github.event_name: ${{ github.event_name }}"' + - run: 'echo "TODO proceed with GHCR upload..."' #- name: Log into registry ${{ env.REGISTRY }} # uses: docker/login-action@v3 From 2e515d99870508c0b2f1b54e820e99cf72eb1ff7 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:28:52 +0000 Subject: [PATCH 59/91] remove TODO --- .github/workflows/ghcr.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 26f054be..66877098 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -14,8 +14,6 @@ on: env: REGISTRY: ghcr.io -# TODO move matrix up here? - jobs: check-images-are-used: # Only run this job if tests pass. See workflow_run definition above. From d49c27369dc7bc560a9e9254a35ef31708f16791 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:31:24 +0000 Subject: [PATCH 60/91] no need --- .github/workflows/ghcr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 66877098..287edc01 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -19,7 +19,6 @@ jobs: # Only run this job if tests pass. See workflow_run definition above. # TODO remove unused parts of this if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - needs: check-images-are-used runs-on: ubuntu-latest permissions: contents: read From b26af10f7c2fb6871e118af9d6089036e85ba491 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:33:42 +0000 Subject: [PATCH 61/91] ref type --- .github/workflows/ghcr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 287edc01..f20dec83 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -42,6 +42,7 @@ jobs: - run: 'echo "- steps.meta.outputs.tags: ${{ steps.meta.outputs.tags }}"' - run: 'echo "- github.event_name: ${{ github.event_name }}"' - run: 'echo "- github.event_name: ${{ github.event_name }}"' + - run: 'echo "- github.ref.type: ${{ github.ref.type }}"' - run: 'echo "- GITHUB_REF: <$GITHUB_REF>"' - run: 'echo "- GITHUB_REF_NAME: <$GITHUB_REF_NAME>"' - run: 'echo "- GITHUB_REF_TYPE: <$GITHUB_REF_TYPE>"' From 16f40d0267b48ddc32015c420ecf797265a784d8 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:34:18 +0000 Subject: [PATCH 62/91] try check only on tag --- .github/workflows/ghcr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index f20dec83..f5f36efa 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -49,6 +49,7 @@ jobs: - name: "Checking release image is referenced in docker-compose.yml..." # TODO only run if tagged + if: ${{ github.ref.type == 'tag' }} run: 'grep "${{ steps.meta.outputs.tags }}" docker-compose.yml' build-push-image: From 2d5d2be159fd37d3bc6cf981cd4697315249eaa6 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:35:50 +0000 Subject: [PATCH 63/91] ref_type --- .github/workflows/ghcr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index f5f36efa..a9b36520 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -43,13 +43,14 @@ jobs: - run: 'echo "- github.event_name: ${{ github.event_name }}"' - run: 'echo "- github.event_name: ${{ github.event_name }}"' - run: 'echo "- github.ref.type: ${{ github.ref.type }}"' + - run: 'echo "- github.ref_type: ${{ github.ref_type }}"' - run: 'echo "- GITHUB_REF: <$GITHUB_REF>"' - run: 'echo "- GITHUB_REF_NAME: <$GITHUB_REF_NAME>"' - run: 'echo "- GITHUB_REF_TYPE: <$GITHUB_REF_TYPE>"' - name: "Checking release image is referenced in docker-compose.yml..." # TODO only run if tagged - if: ${{ github.ref.type == 'tag' }} + if: ${{ github.ref_type == 'tag' }} run: 'grep "${{ steps.meta.outputs.tags }}" docker-compose.yml' build-push-image: From f51b8ac10a61a09453cf351d6d6dc7cc46aacac3 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:50:19 +0000 Subject: [PATCH 64/91] simplify if the second half runs --- .github/workflows/ghcr.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index a9b36520..5cb44bcd 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -49,14 +49,10 @@ jobs: - run: 'echo "- GITHUB_REF_TYPE: <$GITHUB_REF_TYPE>"' - name: "Checking release image is referenced in docker-compose.yml..." - # TODO only run if tagged if: ${{ github.ref_type == 'tag' }} run: 'grep "${{ steps.meta.outputs.tags }}" docker-compose.yml' build-push-image: - # Only run this job if tests pass. See workflow_run definition above. - # TODO remove unused parts of this - if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} needs: check-images-are-used runs-on: ubuntu-latest permissions: From 9a3fd0551472f6f151af40c42a00d70bcbd21332 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:55:35 +0000 Subject: [PATCH 65/91] reinstate tings --- .github/workflows/ghcr.yml | 67 +++++++++----------------- test.yml => .github/workflows/test.yml | 0 2 files changed, 24 insertions(+), 43 deletions(-) rename test.yml => .github/workflows/test.yml (100%) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 5cb44bcd..28397a51 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -1,53 +1,36 @@ name: GHCR on: + workflow_dispatch: # See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_run workflow_run: workflows: [Test] types: [completed] + branches: [master] tags: ["v*.*.*"] - workflow_dispatch: - # disable later: - push: - pull_request: env: REGISTRY: ghcr.io jobs: check-images-are-used: - # Only run this job if tests pass. See workflow_run definition above. - # TODO remove unused parts of this - if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + # Only run this job if tests pass, or this workflow was triggered manually. + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} runs-on: ubuntu-latest permissions: contents: read packages: write - strategy: matrix: image: [nginx, service] - steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Extract Docker metadata id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - - - run: 'echo "TODO Confirm version numbers"' - - run: 'echo "- steps.meta.outputs.tags: ${{ steps.meta.outputs.tags }}"' - - run: 'echo "- github.event_name: ${{ github.event_name }}"' - - run: 'echo "- github.event_name: ${{ github.event_name }}"' - - run: 'echo "- github.ref.type: ${{ github.ref.type }}"' - - run: 'echo "- github.ref_type: ${{ github.ref_type }}"' - - run: 'echo "- GITHUB_REF: <$GITHUB_REF>"' - - run: 'echo "- GITHUB_REF_NAME: <$GITHUB_REF_NAME>"' - - run: 'echo "- GITHUB_REF_TYPE: <$GITHUB_REF_TYPE>"' - - name: "Checking release image is referenced in docker-compose.yml..." if: ${{ github.ref_type == 'tag' }} run: 'grep "${{ steps.meta.outputs.tags }}" docker-compose.yml' @@ -71,33 +54,31 @@ jobs: fetch-tags: true submodules: recursive + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker metadata id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - - run: 'echo "TODO proceed with GHCR upload..."' + - name: Set up QEMU emulator for multi-arch images + uses: docker/setup-qemu-action@v3 - #- name: Log into registry ${{ env.REGISTRY }} - # uses: docker/login-action@v3 - # with: - # registry: ${{ env.REGISTRY }} - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - #- name: Set up QEMU emulator for multi-arch images - # uses: docker/setup-qemu-action@v3 - - #- name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - #- name: Build and push ${{ matrix.image }} Docker image - # uses: docker/build-push-action@v5 - # with: - # file: ${{ matrix.image }}.dockerfile - # context: . - # push: true - # tags: ${{ steps.meta.outputs.tags }} - # labels: ${{ steps.meta.outputs.labels }} - # platforms: 'linux/amd64,linux/arm64' + - name: Build and push ${{ matrix.image }} Docker image + uses: docker/build-push-action@v5 + with: + file: ${{ matrix.image }}.dockerfile + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: 'linux/amd64,linux/arm64' diff --git a/test.yml b/.github/workflows/test.yml similarity index 100% rename from test.yml rename to .github/workflows/test.yml From 4a4ac7ec79a39638fde7a89745d67683f53ebc9a Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:57:30 +0000 Subject: [PATCH 66/91] reinstate test --- .github/workflows/test.yml | 98 +++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0681d764..b4b0b6e3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,49 +1,49 @@ -#name: Test -# -#on: -# push: -# pull_request: -# -#jobs: -# shellcheck: -# timeout-minutes: 2 -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - run: sudo apt-get install shellcheck -# - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | xargs shellcheck --exclude=SC2016 -# test-nginx: -# timeout-minutes: 10 -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# fetch-tags: true -# submodules: recursive -# - uses: actions/setup-node@v4 -# with: -# node-version: 22.12.0 -# - run: cd test/nginx && npm i -# - run: cd test/nginx && ./run-tests.sh -# -# - if: always() -# run: docker logs test-nginx-1 || true -# - if: always() -# run: docker logs test-service-1 || true -# - if: always() -# run: docker logs test-enketo-1 || true -# test-images: -# needs: -# - shellcheck -# - test-nginx -# runs-on: ubuntu-latest # TODO matrix to run on all expected versions? -# steps: -# - uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# fetch-tags: true -# submodules: recursive -# - run: ./test/test-images.sh -# - if: always() -# run: docker compose logs +name: Test + +on: + push: + pull_request: + +jobs: + shellcheck: + timeout-minutes: 2 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: sudo apt-get install shellcheck + - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | xargs shellcheck --exclude=SC2016 + test-nginx: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + - uses: actions/setup-node@v4 + with: + node-version: 22.12.0 + - run: cd test/nginx && npm i + - run: cd test/nginx && ./run-tests.sh + + - if: always() + run: docker logs test-nginx-1 || true + - if: always() + run: docker logs test-service-1 || true + - if: always() + run: docker logs test-enketo-1 || true + test-images: + needs: + - shellcheck + - test-nginx + runs-on: ubuntu-latest # TODO matrix to run on all expected versions? + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + - run: ./test/test-images.sh + - if: always() + run: docker compose logs From 9b8194e360a83f7515948cf4da50282558b6f536 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 10:58:01 +0000 Subject: [PATCH 67/91] revert dockerfile changes --- service.dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/service.dockerfile b/service.dockerfile index 05b188bc..7b947419 100644 --- a/service.dockerfile +++ b/service.dockerfile @@ -53,9 +53,9 @@ RUN apt-get update \ procps \ postgresql-client-14 \ netcat-traditional \ - && rm -rf /var/lib/apt/lists/* -RUN npm clean-install --omit=dev --no-audit \ - --fund=false --update-notifier=false + && rm -rf /var/lib/apt/lists/* \ + && npm clean-install --omit=dev --no-audit \ + --fund=false --update-notifier=false COPY server/ ./ COPY files/service/scripts/ ./ From 69a915c4ff56559217da016dd5287a76350e0297 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 11:00:31 +0000 Subject: [PATCH 68/91] don't build on master --- .github/workflows/ghcr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 28397a51..ec0743ef 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -6,7 +6,6 @@ on: workflow_run: workflows: [Test] types: [completed] - branches: [master] tags: ["v*.*.*"] env: From 27473013c9a204da184ebe3fcfef4769abb691a7 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 11:07:21 +0000 Subject: [PATCH 69/91] run test on tag --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b4b0b6e3..94906918 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,7 @@ name: Test on: push: pull_request: + tags: ["v*.*.*"] jobs: shellcheck: From 08893b7335e50fd71c34664809fbbc60122bb290 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 11:10:02 +0000 Subject: [PATCH 70/91] expicitly run on tags --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 94906918..a603069f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,9 @@ name: Test on: push: + tags: + - v*.*.* pull_request: - tags: ["v*.*.*"] jobs: shellcheck: From c3f255acedf90b0e689084e3725c08a97c607d4b Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 11:22:08 +0000 Subject: [PATCH 71/91] more restruct --- .github/workflows/ghcr.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index ec0743ef..1705ef3a 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -5,8 +5,10 @@ on: # See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_run workflow_run: workflows: [Test] - types: [completed] - tags: ["v*.*.*"] + types: + - completed + tags: + - v*.*.*" env: REGISTRY: ghcr.io From 851adbc0f9ac178c75319346dfd9fa506622a28e Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 30 Jan 2025 11:30:24 +0000 Subject: [PATCH 72/91] rebuild --- rebuild | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 rebuild diff --git a/rebuild b/rebuild new file mode 100644 index 00000000..e69de29b From b48180eb3bb9af38857d870a3d1c540a2da80c14 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 07:01:00 +0000 Subject: [PATCH 73/91] build on pushes (check it still builds on tags...) --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d576018..cf092f66 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,6 @@ name: Test on: push: - tags: - - v*.*.* pull_request: jobs: From a7e5518360ccba1bd46ad8febe18629cd86b7875 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 07:04:08 +0000 Subject: [PATCH 74/91] simplify/revert yaml --- .github/workflows/ghcr.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 1705ef3a..ec0743ef 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -5,10 +5,8 @@ on: # See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_run workflow_run: workflows: [Test] - types: - - completed - tags: - - v*.*.*" + types: [completed] + tags: ["v*.*.*"] env: REGISTRY: ghcr.io From 6e019609f9299b46479c2e9187d039c3d9f2f852 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 07:13:53 +0000 Subject: [PATCH 75/91] Add debug job --- .github/workflows/ghcr.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index ec0743ef..6a4a4057 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -12,6 +12,11 @@ env: REGISTRY: ghcr.io jobs: + debug: + steps: + run: 'echo "github.event_name: [ ${{ github.event_name }} ]"' + run: 'echo "github.event.workflow_run.conclusion: [ ${{ github.event.workflow_run.conclusion }} ]"' + run: "echo 'condition: [ ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} ]'" check-images-are-used: # Only run this job if tests pass, or this workflow was triggered manually. if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} From e653b26c7a0bad7af801ed2152da0d3749dd1afc Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 07:15:26 +0000 Subject: [PATCH 76/91] fix yaml? --- .github/workflows/ghcr.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 6a4a4057..ce99b657 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -13,10 +13,11 @@ env: jobs: debug: + runs-on: ubuntu-latest steps: - run: 'echo "github.event_name: [ ${{ github.event_name }} ]"' - run: 'echo "github.event.workflow_run.conclusion: [ ${{ github.event.workflow_run.conclusion }} ]"' - run: "echo 'condition: [ ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} ]'" + - run: 'echo "github.event_name: [ ${{ github.event_name }} ]"' + - run: 'echo "github.event.workflow_run.conclusion: [ ${{ github.event.workflow_run.conclusion }} ]"' + - run: "echo 'condition: [ ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} ]'" check-images-are-used: # Only run this job if tests pass, or this workflow was triggered manually. if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} From d66ac83023e947ff2671284972e245ee1e0c76d4 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 07:17:02 +0000 Subject: [PATCH 77/91] stub out test jobs --- .github/workflows/test.yml | 43 ++------------------------------------ 1 file changed, 2 insertions(+), 41 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf092f66..8476909c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,46 +5,7 @@ on: pull_request: jobs: - test-misc: # quick, simple checks - timeout-minutes: 2 + dummy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - run: sudo apt-get install shellcheck - - run: ./test/check-scripts.sh - - run: ./test/check-dockerfiles.sh - test-nginx: - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - submodules: recursive - - uses: actions/setup-node@v4 - with: - node-version: 22.12.0 - - run: cd test/nginx && npm i - - run: cd test/nginx && ./run-tests.sh - - - if: always() - run: docker logs test-nginx-1 || true - - if: always() - run: docker logs test-service-1 || true - - if: always() - run: docker logs test-enketo-1 || true - test-images: - needs: - - test-misc - - test-nginx - runs-on: ubuntu-latest # TODO matrix to run on all expected versions? - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - submodules: recursive - - run: ./test/test-images.sh - - if: always() - run: docker compose logs + - run: echo hi From ee1a2ad8a44e2f22e9638c700f9b9d89ebeeb628 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 07:22:08 +0000 Subject: [PATCH 78/91] remove rebuild --- rebuild | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 rebuild diff --git a/rebuild b/rebuild deleted file mode 100644 index e69de29b..00000000 From 1fc807beb675b4349cc03a3fa8c7f6ebf196d893 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 07:56:02 +0000 Subject: [PATCH 79/91] Add make release target --- Makefile | 8 ++++++++ docker-compose.yml | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dd192f47..60d93103 100644 --- a/Makefile +++ b/Makefile @@ -7,3 +7,11 @@ dev: .PHONY: stop stop: $(docker-compose-dev) stop + +.PHONY: release +release: + @printf "Enter version number: "; \ + read VERSION; \ + sed -E -e "s/{{SNAPSHOT}}/$$VERSION/" docker-compose.yml; \ + git tag "$$VERSION"; \ + git push --tags diff --git a/docker-compose.yml b/docker-compose.yml index 56513718..5e5717b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: - DKIM_KEY_PATH=/etc/exim4/dkim.key.temp restart: always service: - image: 'ghcr.io/getodk/central-service:2025.1' + image: 'ghcr.io/getodk/central-service:{{SNAPSHOT}}' depends_on: - secrets - postgres14 @@ -77,7 +77,7 @@ services: logging: driver: local nginx: - image: 'ghcr.io/getodk/central-nginx:2025.1' + image: 'ghcr.io/getodk/central-nginx:{{SNAPSHOT}}' depends_on: - service - enketo From acee5d533117ef337269bf247ad34dc4c3ecd2d9 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 07:57:18 +0000 Subject: [PATCH 80/91] Revert "Add make release target" This reverts commit 1fc807beb675b4349cc03a3fa8c7f6ebf196d893. --- Makefile | 8 -------- docker-compose.yml | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 60d93103..dd192f47 100644 --- a/Makefile +++ b/Makefile @@ -7,11 +7,3 @@ dev: .PHONY: stop stop: $(docker-compose-dev) stop - -.PHONY: release -release: - @printf "Enter version number: "; \ - read VERSION; \ - sed -E -e "s/{{SNAPSHOT}}/$$VERSION/" docker-compose.yml; \ - git tag "$$VERSION"; \ - git push --tags diff --git a/docker-compose.yml b/docker-compose.yml index 5e5717b8..56513718 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: - DKIM_KEY_PATH=/etc/exim4/dkim.key.temp restart: always service: - image: 'ghcr.io/getodk/central-service:{{SNAPSHOT}}' + image: 'ghcr.io/getodk/central-service:2025.1' depends_on: - secrets - postgres14 @@ -77,7 +77,7 @@ services: logging: driver: local nginx: - image: 'ghcr.io/getodk/central-nginx:{{SNAPSHOT}}' + image: 'ghcr.io/getodk/central-nginx:2025.1' depends_on: - service - enketo From 08b6d8235325d9007af6d6e803e31ff3e10c1ba6 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 07:58:41 +0000 Subject: [PATCH 81/91] import ghcr into test --- .github/workflows/test.yml | 81 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8476909c..67aed995 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,8 +4,89 @@ on: push: pull_request: +env: + REGISTRY: ghcr.io + jobs: dummy: runs-on: ubuntu-latest steps: - run: echo hi + debug: + runs-on: ubuntu-latest + steps: + - run: 'echo "github.event_name: [ ${{ github.event_name }} ]"' + - run: 'echo "github.event_name: [ ${{ github.ref_type }} ]"' + - run: 'echo "github.event.workflow_run.conclusion: [ ${{ github.event.workflow_run.conclusion }} ]"' + - run: 'echo "Tag name from GITHUB_REF_NAME: $GITHUB_REF_NAME"' + - run: 'echo "Tag name from github.ref_name: ${{ github.ref_name }}"' + - run: "echo 'condition 1: [ ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} ]'" + - run: "echo 'condition 2: [ ${{ github.ref_type == 'tag' }} && (${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}) ]'" + check-images-are-used: + if: ${{ github.ref_type == 'tag' }} + needs: [dummy] + runs-on: ubuntu-latest + permissions: # remove permissions + contents: read + packages: write + strategy: + matrix: + image: [nginx, service] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} + - name: "Checking release image is referenced in docker-compose.yml..." + run: 'grep "${{ steps.meta.outputs.tags }}" docker-compose.yml' + + build-push-image: + needs: check-images-are-used + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + image: [nginx, service] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} + + - name: Set up QEMU emulator for multi-arch images + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push ${{ matrix.image }} Docker image + uses: docker/build-push-action@v5 + with: + file: ${{ matrix.image }}.dockerfile + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: 'linux/amd64,linux/arm64' From d0938bae1171eddf0671a8efbd90c3be8279c625 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 07:59:02 +0000 Subject: [PATCH 82/91] remove ghcr workflow --- .github/workflows/ghcr.yml | 89 -------------------------------------- 1 file changed, 89 deletions(-) delete mode 100644 .github/workflows/ghcr.yml diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml deleted file mode 100644 index ce99b657..00000000 --- a/.github/workflows/ghcr.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: GHCR - -on: - workflow_dispatch: - # See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_run - workflow_run: - workflows: [Test] - types: [completed] - tags: ["v*.*.*"] - -env: - REGISTRY: ghcr.io - -jobs: - debug: - runs-on: ubuntu-latest - steps: - - run: 'echo "github.event_name: [ ${{ github.event_name }} ]"' - - run: 'echo "github.event.workflow_run.conclusion: [ ${{ github.event.workflow_run.conclusion }} ]"' - - run: "echo 'condition: [ ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} ]'" - check-images-are-used: - # Only run this job if tests pass, or this workflow was triggered manually. - if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - matrix: - image: [nginx, service] - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - - name: "Checking release image is referenced in docker-compose.yml..." - if: ${{ github.ref_type == 'tag' }} - run: 'grep "${{ steps.meta.outputs.tags }}" docker-compose.yml' - - build-push-image: - needs: check-images-are-used - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - strategy: - matrix: - image: [nginx, service] - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - submodules: recursive - - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - - - name: Set up QEMU emulator for multi-arch images - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push ${{ matrix.image }} Docker image - uses: docker/build-push-action@v5 - with: - file: ${{ matrix.image }}.dockerfile - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: 'linux/amd64,linux/arm64' From 9abf42ca02930f5a5e984183e9c70d0e17cf562a Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 08:06:24 +0000 Subject: [PATCH 83/91] re-intro standard CI --- .github/workflows/test.yml | 141 +++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 59 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67aed995..e1433bb0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,23 +8,53 @@ env: REGISTRY: ghcr.io jobs: - dummy: + test-misc: # quick, simple checks + timeout-minutes: 2 runs-on: ubuntu-latest steps: - - run: echo hi - debug: + - uses: actions/checkout@v4 + - run: sudo apt-get install shellcheck + - run: ./test/check-scripts.sh + - run: ./test/check-dockerfiles.sh + test-nginx: + timeout-minutes: 10 runs-on: ubuntu-latest steps: - - run: 'echo "github.event_name: [ ${{ github.event_name }} ]"' - - run: 'echo "github.event_name: [ ${{ github.ref_type }} ]"' - - run: 'echo "github.event.workflow_run.conclusion: [ ${{ github.event.workflow_run.conclusion }} ]"' - - run: 'echo "Tag name from GITHUB_REF_NAME: $GITHUB_REF_NAME"' - - run: 'echo "Tag name from github.ref_name: ${{ github.ref_name }}"' - - run: "echo 'condition 1: [ ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} ]'" - - run: "echo 'condition 2: [ ${{ github.ref_type == 'tag' }} && (${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}) ]'" + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + - uses: actions/setup-node@v4 + with: + node-version: 22.12.0 + - run: cd test/nginx && npm i + - run: cd test/nginx && ./run-tests.sh + + - if: always() + run: docker logs test-nginx-1 || true + - if: always() + run: docker logs test-service-1 || true + - if: always() + run: docker logs test-enketo-1 || true + test-images: + needs: + - test-misc + - test-nginx + runs-on: ubuntu-latest # TODO matrix to run on all expected versions? + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + - run: ./test/test-images.sh + - if: always() + run: docker compose logs check-images-are-used: if: ${{ github.ref_type == 'tag' }} - needs: [dummy] + needs: + - test-images runs-on: ubuntu-latest permissions: # remove permissions contents: read @@ -33,60 +63,53 @@ jobs: matrix: image: [nginx, service] steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - - name: "Checking release image is referenced in docker-compose.yml..." - run: 'grep "${{ steps.meta.outputs.tags }}" docker-compose.yml' - + - name: Checkout repository + uses: actions/checkout@v4 + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} + - name: Check release image is referenced in docker-compose.yml + run: 'grep "${{ steps.meta.outputs.tags }}" docker-compose.yml' build-push-image: - needs: check-images-are-used + needs: + - check-images-are-used runs-on: ubuntu-latest permissions: contents: read packages: write - strategy: matrix: image: [nginx, service] - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - submodules: recursive - - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - - - name: Set up QEMU emulator for multi-arch images - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push ${{ matrix.image }} Docker image - uses: docker/build-push-action@v5 - with: - file: ${{ matrix.image }}.dockerfile - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: 'linux/amd64,linux/arm64' + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} + - name: Set up QEMU emulator for multi-arch images + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push ${{ matrix.image }} Docker image + uses: docker/build-push-action@v5 + with: + file: ${{ matrix.image }}.dockerfile + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: 'linux/amd64,linux/arm64' From 97cf7327ed2cd5ddfa26d80f155ddf7964d3e3df Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 08:07:28 +0000 Subject: [PATCH 84/91] remove extra permissiosn --- .github/workflows/test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1433bb0..19aa3d97 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,9 +56,6 @@ jobs: needs: - test-images runs-on: ubuntu-latest - permissions: # remove permissions - contents: read - packages: write strategy: matrix: image: [nginx, service] From 5750c0a37d2495d65bcb9a4836bd2d0513ab9d24 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 08:27:47 +0000 Subject: [PATCH 85/91] add tag check --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 19aa3d97..9065ff30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,6 +60,8 @@ jobs: matrix: image: [nginx, service] steps: + - name: Check tag format + run: '[[ "${{ github.ref_name }}" = v*.*.* ]]' - name: Checkout repository uses: actions/checkout@v4 - name: Extract Docker metadata From 08d168abab77306bc70b68009e2e592d90f1d127 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 08:34:16 +0000 Subject: [PATCH 86/91] add release script --- release.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 release.sh diff --git a/release.sh b/release.sh new file mode 100755 index 00000000..5a059989 --- /dev/null +++ b/release.sh @@ -0,0 +1,27 @@ +#!/bin/bash -eu +set -o pipefail + +log() { echo "[release] $*"; } + +# TODO check git is clean & up-to-date with remote +# TODO try to divine version +# TODO confirm version with user +# TODO check entered version format is valid + +sed -E \ + -e "s_'ghcr.io/getodk/central-nginx:.*'_'ghcr.io/getodk/central-nginx:$newVersion'_" \ + -e "s_'ghcr.io/getodk/central-service:.*'_'ghcr.io/getodk/central-service:$newVersion'_" \ + docker-compose.yml > "$tmpfile" +mv "$tmpfile" docker-compose.yml + +git add docker-compose.yml +git commit -m"Release version: $newVersion" +git tag "$newVersion" + +log "Pushing release to git..." +git push && git push --tags + +log "Release complete. Check build progress at:" +log "" +log " https://github.com/getodk/central/actions" +echo From 814d416e11e9d16d728c5ad4a69533bf342ff136 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 08:47:55 +0000 Subject: [PATCH 87/91] add release script --- release.sh | 62 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/release.sh b/release.sh index 5a059989..ad332b24 100755 --- a/release.sh +++ b/release.sh @@ -3,17 +3,69 @@ set -o pipefail log() { echo "[release] $*"; } -# TODO check git is clean & up-to-date with remote -# TODO try to divine version -# TODO confirm version with user -# TODO check entered version format is valid +log "Checking git branch..." +currentBranch="$(git rev-parse --abbrev-ref HEAD)" +if [[ "$currentBranch" != master ]]; then + log "!!!" + log "!!! Unexpected branch:" + log "!!! Expected: master" + log "!!! Actual: $currentBranch" + log "!!!" + exit 1 +fi +log " Git branch OK." +log "Checking for uncommitted changes..." +gitStatus="$(git status --porcelain)" +if [[ "$gitStatus" != "" ]]; then + log "!!!" + log "!!! Your working directory is dirty. Make sure you have committed all changes." + log "!!!" + exit 1 +fi +log " No uncommitted changes found." + +log "Checking for divergence from upstream..." +upstream="$(git rev-parse --abbrev-ref '@{upstream}')" +remote="${upstream%%/*}" +log " Fetching from remote '$remote'..." +git fetch "$remote" +log " Comparing to $upstream..." +if ! git diff --exit-code "$upstream"..HEAD; then + log "!!!" + log "!!! Differences found between HEAD and tracking branch $upstream !!!" + log "!!!" + log "!!! Do you need to git push?" + log "!!!" + exit 1 +fi +log " HEAD seems to be in-line with upstream." + +year="$(date +%Y)" +if git tag | grep "^$year\."; then + lastMinor="$(git tag | grep v2024 | tail -n1 | cut -d'.' -f2)" + suggestedVersion="$year.$((lastMinor+1)).0" +else + suggestedVersion="$year.1.0" +fi +printf "[release] Version to release ($suggestedVersion): " +read -p newVersion +if ! [[ "$newVersion" = v*.*.* ]]; then + log "!!!" + log "!!! Version '$newVersion' does not match expected format." + log "!!!" + exit 1 +fi + +log "Updating version numbers in docker-compose.yml ..." +tmpfile="$(mktemp)" sed -E \ -e "s_'ghcr.io/getodk/central-nginx:.*'_'ghcr.io/getodk/central-nginx:$newVersion'_" \ -e "s_'ghcr.io/getodk/central-service:.*'_'ghcr.io/getodk/central-service:$newVersion'_" \ - docker-compose.yml > "$tmpfile" + docker-compose.yml > "$tmpfile" mv "$tmpfile" docker-compose.yml +log "Committing changes to git..." git add docker-compose.yml git commit -m"Release version: $newVersion" git tag "$newVersion" From f5dc805005952149e2db1c321b8f5ae78c9ff7a4 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 09:00:52 +0000 Subject: [PATCH 88/91] tweak script --- release.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/release.sh b/release.sh index ad332b24..b9fddfb9 100755 --- a/release.sh +++ b/release.sh @@ -42,14 +42,13 @@ fi log " HEAD seems to be in-line with upstream." year="$(date +%Y)" -if git tag | grep "^$year\."; then - lastMinor="$(git tag | grep v2024 | tail -n1 | cut -d'.' -f2)" - suggestedVersion="$year.$((lastMinor+1)).0" +if git tag | grep "^v$year\."; then + lastMinor="$(git tag | grep "^v$year" | tail -n1 | cut -d'.' -f2)" + suggestedVersion="v$year.$((lastMinor+1)).0" else - suggestedVersion="$year.1.0" + suggestedVersion="v$year.1.0" fi -printf "[release] Version to release ($suggestedVersion): " -read -p newVersion +read -e -p "[release] Version to release: " -i "$suggestedVersion" newVersion if ! [[ "$newVersion" = v*.*.* ]]; then log "!!!" log "!!! Version '$newVersion' does not match expected format." @@ -60,8 +59,8 @@ fi log "Updating version numbers in docker-compose.yml ..." tmpfile="$(mktemp)" sed -E \ - -e "s_'ghcr.io/getodk/central-nginx:.*'_'ghcr.io/getodk/central-nginx:$newVersion'_" \ - -e "s_'ghcr.io/getodk/central-service:.*'_'ghcr.io/getodk/central-service:$newVersion'_" \ + -e "s_(image:\s+'.*/.*/central-nginx):.*'_\1:$newVersion'_" \ + -e "s_(image:\s+'.*/.*/central-service):.*'_\1:$newVersion'_" \ docker-compose.yml > "$tmpfile" mv "$tmpfile" docker-compose.yml @@ -73,6 +72,7 @@ git tag "$newVersion" log "Pushing release to git..." git push && git push --tags +echo log "Release complete. Check build progress at:" log "" log " https://github.com/getodk/central/actions" From 0b22c62354a3405fc1394ebac3a7740e364410c9 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 09:05:30 +0000 Subject: [PATCH 89/91] shellcheck --- release.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release.sh b/release.sh index b9fddfb9..9c1d95fa 100755 --- a/release.sh +++ b/release.sh @@ -48,7 +48,9 @@ if git tag | grep "^v$year\."; then else suggestedVersion="v$year.1.0" fi -read -e -p "[release] Version to release: " -i "$suggestedVersion" newVersion +read -r -e \ + -p "[release] Version to release: " \ + -i "$suggestedVersion" newVersion if ! [[ "$newVersion" = v*.*.* ]]; then log "!!!" log "!!! Version '$newVersion' does not match expected format." From ed2c1416e8fd171ec7ad116b3e8a710ebf217380 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 14 Feb 2025 09:30:50 +0000 Subject: [PATCH 90/91] ci: rename check-image-references job --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9065ff30..7728b2b2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,7 +51,7 @@ jobs: - run: ./test/test-images.sh - if: always() run: docker compose logs - check-images-are-used: + check-image-references: if: ${{ github.ref_type == 'tag' }} needs: - test-images @@ -73,7 +73,7 @@ jobs: run: 'grep "${{ steps.meta.outputs.tags }}" docker-compose.yml' build-push-image: needs: - - check-images-are-used + - check-image-references runs-on: ubuntu-latest permissions: contents: read From c6931634ee26c54b0f47e86f70bd34b1ff8f79e9 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 20 Feb 2025 08:21:44 +0000 Subject: [PATCH 91/91] use existing version --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 56513718..1c235577 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: - DKIM_KEY_PATH=/etc/exim4/dkim.key.temp restart: always service: - image: 'ghcr.io/getodk/central-service:2025.1' + image: 'ghcr.io/getodk/central-service:v2024.3.1' depends_on: - secrets - postgres14 @@ -77,7 +77,7 @@ services: logging: driver: local nginx: - image: 'ghcr.io/getodk/central-nginx:2025.1' + image: 'ghcr.io/getodk/central-nginx:v2024.3.1' depends_on: - service - enketo