diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e717212f76..53745a119e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,7 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} + NODE_VERSION: 18 permissions: packages: write @@ -18,16 +19,31 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install dependencies + run: yarn install --frozen-lockfile + env: + NODE_OPTIONS: "--max_old_space_size=4096" + + - name: Build TerriaMap + run: yarn gulp lint release + env: + NODE_OPTIONS: "--max_old_space_size=4096" - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to GHCR - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -35,15 +51,13 @@ jobs: - name: Tag image id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | type=semver,pattern={{version}} + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index - name: Build and push - uses: docker/build-push-action@v2 - with: - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - push: true + run: yarn docker-build-prod --metadata diff --git a/.prettierignore b/.prettierignore index 825a640036..4ca578014e 100644 --- a/.prettierignore +++ b/.prettierignore @@ -43,4 +43,4 @@ wwwroot/*.html ckanext-cesiumpreview -deploy/helm/terria/charts/terriamap/templates/ \ No newline at end of file +deploy/helm/terriamap/templates/ diff --git a/architecture/0001-npm-lockfiles.md b/architecture/0001-npm-lockfiles.md index aee2b20612..2cc9932102 100644 --- a/architecture/0001-npm-lockfiles.md +++ b/architecture/0001-npm-lockfiles.md @@ -4,7 +4,7 @@ Date: 2020-03-02 ## Status -Accepted +Superseded 2021-10-09 by decision to use yarn everywhere. ## Context diff --git a/architecture/0002-docker-multi-arch-build.md b/architecture/0002-docker-multi-arch-build.md new file mode 100644 index 0000000000..f1f54f43f4 --- /dev/null +++ b/architecture/0002-docker-multi-arch-build.md @@ -0,0 +1,56 @@ +# 2. Docker multi-architecture build using create-docker-context + +Date: 2024-07-18 + +## Status + +Proposed + +## Context + +I (crispy) consider a multi-stage dockerfile to be the gold standard of +reproducible, mutli-architecture builds. Something like the following: + +- Build container copies workspace, installs development dependencies and builds + the app. +- Production container copies build artifacts and installs only production + dependencies. + +is ideal. This ensures only production dependencies are present, and you can run +this process on every architecture to create a multi-arch docker image. Binaries +downloaded during dependency installation will fetch the correct architecture +binary as depencies are installed separately on each architecture. But +installing dependencies and building JS is extremely slow on emulated +architectures, such as docker buildx on GitHub Actions (this can take 2.5 hours +to build the image). + +If instead we can (on the build machine/VM): + +1. install all dependencies +2. build the app +3. copy build artifacts and only production dependencies to the multi-arch + docker image + +then **as long as production dependencies are portable**, we have a working +multi-arch docker image with very little computation being run on emulated +architectures. The `create-docker-context.js` script allows us to do this, +copying build artifacts and only production dependencies to an intermediate +"context" folder which is then used to create the image. + +Currently none of our production dependencies install non-portable binaries. + +## Decision + +While all production dependencies remain portable, we will build +multi-architecture docker images by building TerriaMap on the VM and copying +only production-necessary files and dependencies to the final docker image. + +## Consequences + +- We will replace current GitHub Actions release process with one using + `create-docker-context.js`. +- Our GitHub Actions TerriaMap release time will reduce from 2.5 hours to less + than 10 minutes. +- If in future TerriaMap uses a binary installed by side effect during JS + dependency installation and this binary cannot be run on an architecture for + which an image is created, that image will fail when run on that architecture. diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 90b67ac7ca..3b8317fe49 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,9 +1,12 @@ # Docker image for the primary terria map application server -FROM node:16 +# Intended for use only with a "context" directory created by create-docker-context.js +FROM node:16-slim -RUN mkdir -p /usr/src/app && mkdir -p /etc/config/client -WORKDIR /usr/src/app/component -COPY . /usr/src/app +RUN mkdir -p /etc/config/client + +USER node +WORKDIR /usr/src/app +COPY --chown=node:node component /usr/src/app EXPOSE 3001 ENV NODE_ENV=production diff --git a/deploy/docker/create-docker-context.js b/deploy/docker/create-docker-context.js old mode 100644 new mode 100755 index d04d248c57..f9847c3152 --- a/deploy/docker/create-docker-context.js +++ b/deploy/docker/create-docker-context.js @@ -1,8 +1,16 @@ #!/usr/bin/env node +// MAJOR ASSSUMPTION: build artifacts and node_modules content for all production +// dependencies is cross-platform, or care is taken to only install dependencies, +// run create-docker-context.js and create docker images on a compatible platform +// See architecture/0002-docker-multi-arch-build.md + // Based off @magda/docker-utils@2.1.0 create-docker-context-for-node-component // Changes made: -// - The Dockerfile path is configurable in package.json +// - The Dockerfile path is configurable in package.json (I don't want a dockerfile +// intended to be used only through a script to be in the top level directory) +// - Can parse metadata from GitHub Action docker/metadata-action@v5 and add this +// to the created image const childProcess = require("child_process"); const fse = require("fs-extra"); @@ -42,7 +50,7 @@ const argv = yargs }, name: { description: - "The package name to use in auto tag generation. Will default to ''. Used to override the docker nanme config in package.json during the auto tagging. Requires --tag=auto", + "The package name to use in auto tag generation. Will default to ''. Used to override the docker name config in package.json during the auto tagging. Requires --tag=auto", type: "string", default: process.env.MAGDA_DOCKER_NAME }, @@ -86,6 +94,12 @@ const argv = yargs description: "Version to cache from when building, using the --cache-from field in docker. Will use the same repository and name. Using this options causes the image to be pulled before build.", type: "string" + }, + metadata: { + description: + "Use tags and annotations from https://github.com/docker/metadata-action v5. Utilises env.DOCKER_METADATA_OUTPUT_JSON. Overrides --tag", + type: "boolean", + default: false } }) .help().argv; @@ -166,16 +180,22 @@ if (argv.build) { } ); - const tags = getTags( - argv.tag, - argv.local, - argv.repository, - argv.version, - argv.name - ); - const tagArgs = tags - .map((tag) => ["-t", tag]) - .reduce((soFar, tagArgs) => soFar.concat(tagArgs), []); + // metadata json from GitHub Action docker/metadata-action@v5 + const metadata = + argv.metadata && env.DOCKER_METADATA_OUTPUT_JSON + ? JSON.parse(env.DOCKER_METADATA_OUTPUT_JSON) + : undefined; + + const tags = metadata + ? metadata.tags + : getTags(argv.tag, argv.local, argv.repository, argv.version, argv.name); + const tagArgs = tags.flatMap((tag) => ["-t", tag]); + + const annotationArgs = + metadata?.annotations?.flatMap((annotation) => [ + "--annotation", + annotation + ]) ?? []; const cacheFromArgs = cacheFromImage ? ["--cache-from", cacheFromImage] : []; @@ -186,6 +206,7 @@ if (argv.build) { ...(argv.platform ? ["buildx"] : []), "build", ...tagArgs, + ...annotationArgs, ...cacheFromArgs, ...(argv.noCache ? ["--no-cache"] : []), ...(argv.platform ? ["--platform", argv.platform, "--push"] : []), diff --git a/deploy/helm/terria/Chart.yaml b/deploy/helm/terria/Chart.yaml deleted file mode 100644 index 7a97f32415..0000000000 --- a/deploy/helm/terria/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -description: "An open source geospatial data explorer" -name: "terria" -version: "0.1.0" -home: "https://github.com/TerriaJS/terriamap" -sources: ["https://github.com/TerriaJS/terriamap"] diff --git a/deploy/helm/terria/charts/terriamap/Chart.yaml b/deploy/helm/terria/charts/terriamap/Chart.yaml deleted file mode 100644 index 96fec5f15b..0000000000 --- a/deploy/helm/terria/charts/terriamap/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: A Helm chart for Kubernetes -name: terriamap -version: 0.1.0 diff --git a/deploy/helm/terria/values.yaml b/deploy/helm/terria/values.yaml deleted file mode 100644 index 9f1b1ecf3d..0000000000 --- a/deploy/helm/terria/values.yaml +++ /dev/null @@ -1,9 +0,0 @@ -global: - rollingUpdate: - maxUnavailable: 0 - exposeNodePorts: false - image: - repository: "ghcr.io/terriajs" - pullPolicy: Always -tags: - all: diff --git a/deploy/helm/terria/.helmignore b/deploy/helm/terriamap/.helmignore similarity index 96% rename from deploy/helm/terria/.helmignore rename to deploy/helm/terriamap/.helmignore index f0c1319444..22a0f43ad9 100644 --- a/deploy/helm/terria/.helmignore +++ b/deploy/helm/terriamap/.helmignore @@ -1,6 +1,7 @@ # Patterns to ignore when building packages. # This supports shell glob matching, relative path matching, and # negation (prefixed with !). Only one pattern per line. +.helmignore .DS_Store # Common VCS dirs .git/ diff --git a/deploy/helm/terriamap/Chart.yaml b/deploy/helm/terriamap/Chart.yaml new file mode 100644 index 0000000000..894f4a5be0 --- /dev/null +++ b/deploy/helm/terriamap/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +description: A Helm chart for Kubernetes +name: terriamap +version: 0.1.2 +home: https://github.com/TerriaJS/terriamap +sources: ["https://github.com/TerriaJS/terriamap"] diff --git a/deploy/helm/terria/requirements.yaml b/deploy/helm/terriamap/requirements.yaml similarity index 100% rename from deploy/helm/terria/requirements.yaml rename to deploy/helm/terriamap/requirements.yaml diff --git a/deploy/helm/terria/templates/_helpers.tpl b/deploy/helm/terriamap/templates/_helpers.tpl similarity index 100% rename from deploy/helm/terria/templates/_helpers.tpl rename to deploy/helm/terriamap/templates/_helpers.tpl diff --git a/deploy/helm/terria/charts/terriamap/templates/configmap-client.yaml b/deploy/helm/terriamap/templates/configmap-client.yaml similarity index 100% rename from deploy/helm/terria/charts/terriamap/templates/configmap-client.yaml rename to deploy/helm/terriamap/templates/configmap-client.yaml diff --git a/deploy/helm/terria/charts/terriamap/templates/configmap-server.yaml b/deploy/helm/terriamap/templates/configmap-server.yaml similarity index 100% rename from deploy/helm/terria/charts/terriamap/templates/configmap-server.yaml rename to deploy/helm/terriamap/templates/configmap-server.yaml diff --git a/deploy/helm/terria/charts/terriamap/templates/deployment.yaml b/deploy/helm/terriamap/templates/deployment.yaml similarity index 96% rename from deploy/helm/terria/charts/terriamap/templates/deployment.yaml rename to deploy/helm/terriamap/templates/deployment.yaml index 5d5bcb47d7..fa6b65391e 100644 --- a/deploy/helm/terria/charts/terriamap/templates/deployment.yaml +++ b/deploy/helm/terriamap/templates/deployment.yaml @@ -31,7 +31,7 @@ spec: - name: terriamap-config-server mountPath: /etc/config/server - name: terriamap-config-client - mountPath: /usr/src/app/component/wwwroot/config.json + mountPath: /usr/src/app/wwwroot/config.json subPath: config.json volumes: - name: terriamap-config-client diff --git a/deploy/helm/terria/charts/terriamap/templates/service.yaml b/deploy/helm/terriamap/templates/service.yaml similarity index 100% rename from deploy/helm/terria/charts/terriamap/templates/service.yaml rename to deploy/helm/terriamap/templates/service.yaml diff --git a/deploy/helm/terria/charts/terriamap/values.yaml b/deploy/helm/terriamap/values.yaml similarity index 90% rename from deploy/helm/terria/charts/terriamap/values.yaml rename to deploy/helm/terriamap/values.yaml index bd3004d273..e8ef61aab5 100644 --- a/deploy/helm/terria/charts/terriamap/values.yaml +++ b/deploy/helm/terriamap/values.yaml @@ -1,12 +1,20 @@ +global: + rollingUpdate: + maxUnavailable: 0 + exposeNodePorts: false + image: + repository: + tags: + pullPolicy: nodePort: image: # By default this pulls ghcr.io/terriajs/terrimap:latest # Set "full" to specify a custom terriamap image to be used # Or you can set "repository" or "tag" if required # full: "ghcr.io/terriajs/terriamap:0.1.1" - # repository: ghcr.io/terriajs + repository: ghcr.io/terriajs # tag: latest - pullPolicy: Always + pullPolicy: IfNotPresent clientConfig: initializationUrls: - helm diff --git a/package.json b/package.json index b2a915fb0c..648d2552e8 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ ] }, "name": "terriajs-map", - "version": "0.1.2", + "version": "0.2.0-alpha.4", "description": "Geospatial catalog explorer based on TerriaJS.", "license": "Apache-2.0", "engines": { @@ -96,7 +96,7 @@ "scripts": { "prepare": "husky install", "docker-build-local": "node deploy/docker/create-docker-context.js --build --push --tag auto --local", - "docker-build-prod": "node deploy/docker/create-docker-context.js --build --push --tag auto --repository=ghcr.io/terriajs", + "docker-build-prod": "node deploy/docker/create-docker-context.js --build --push --platform=linux/amd64,linux/arm64", "docker-build-ci": "node deploy/docker/create-docker-context.js --build", "start": "terriajs-server --config-file serverconfig.json", "gulp": "gulp",