Release TerriaMap using create-docker-context

.github/workflows/release.yml

@@ -8,6 +8,7 @@ on:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
+  NODE_VERSION: 18
 
 permissions:
   packages: write
@@ -18,32 +19,45 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+        env:
+          NODE_OPTIONS: "--max_old_space_size=4096"
+
+      - name: Build TerriaMap
+        run: yarn gulp lint release
+        env:
+          NODE_OPTIONS: "--max_old_space_size=4096"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to GHCR
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Tag image
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=semver,pattern={{version}}
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
 
       - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.meta.outputs.tags }}
-          push: true
+        run: yarn docker-build-prod --metadata

architecture/0001-npm-lockfiles.md

@@ -4,7 +4,7 @@ Date: 2020-03-02
 
 ## Status
 
-Accepted
+Superseded 2021-10-09 by decision to use yarn everywhere.
 
 ## Context
 

architecture/0002-docker-multi-arch-build.md

@@ -0,0 +1,56 @@
+# 2. Docker multi-architecture build using create-docker-context
+
+Date: 2024-07-18
+
+## Status
+
+Proposed
+
+## Context
+
+I (crispy) consider a multi-stage dockerfile to be the gold standard of
+reproducible, mutli-architecture builds. Something like the following:
+
+- Build container copies workspace, installs development dependencies and builds
+  the app.
+- Production container copies build artifacts and installs only production
+  dependencies.
+
+is ideal. This ensures only production dependencies are present, and you can run
+this process on every architecture to create a multi-arch docker image. Binaries
+downloaded during dependency installation will fetch the correct architecture
+binary as depencies are installed separately on each architecture. But
+installing dependencies and building JS is extremely slow on emulated
+architectures, such as docker buildx on GitHub Actions (this can take 2.5 hours
+to build the image).
+
+If instead we can (on the build machine/VM):
+
+1. install all dependencies
+2. build the app
+3. copy build artifacts and only production dependencies to the multi-arch
+   docker image
+
+then **as long as production dependencies are portable**, we have very little
+computation being run on emulated architectures. The `create-docker-context.js`
+script allows us to do this, copying build artifacts and only production
+dependencies to an intermediate "context" folder which is then used to create
+the
+
+Currently none of our production dependencies install non-portable binaries.
+
+## Decision
+
+While all production dependencies remain portable, we will build
+multi-architecture docker images by building TerriaMap on the VM and copying
+only production-necessary files and dependencies to the final docker image.
+
+## Consequences
+
+- We will replace current GitHub Actions release process with one using
+  `create-docker-context.js`.
+- Our GitHub Actions TerriaMap release time will reduce from 2.5 hours to less
+  than 10 minutes.
+- If in future TerriaMap uses a binary installed by side effect during JS
+  dependency installation and this binary cannot be run on an architecture for
+  which an image is created, that image will fail when run on that architecture.

deploy/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Docker image for the primary terria map application server
-FROM node:16
+FROM node:16-slim
 
 RUN mkdir -p /usr/src/app && mkdir -p /etc/config/client
 WORKDIR /usr/src/app/component

deploy/docker/create-docker-context.js

@@ -1,8 +1,16 @@
 #!/usr/bin/env node
 
+// MAJOR ASSSUMPTION: build artifacts and node_modules content for all production
+// dependencies is cross-platform, or care is taken to only install dependencies,
+// run create-docker-context.js and create docker images on a compatible platform
+// See architecture/0002-docker-multi-arch-build.md
+
 // Based off @magda/docker-utils@2.1.0 create-docker-context-for-node-component
 // Changes made:
-// - The Dockerfile path is configurable in package.json
+// - The Dockerfile path is configurable in package.json (I don't want a dockerfile
+//    intended to be used only through a script to be in the top level directory)
+// - Can parse metadata from GitHub Action docker/metadata-action@v5 and add this
+//    to the created image
 
 const childProcess = require("child_process");
 const fse = require("fs-extra");
@@ -86,6 +94,12 @@ const argv = yargs
       description:
         "Version to cache from when building, using the --cache-from field in docker. Will use the same repository and name. Using this options causes the image to be pulled before build.",
       type: "string"
+    },
+    metadata: {
+      description:
+        "Use tags and annotations from https://github.com/docker/metadata-action v5. Utilises env.DOCKER_METADATA_OUTPUT_JSON. Overrides --tag",
+      type: "boolean",
+      default: false
     }
   })
   .help().argv;
@@ -166,16 +180,22 @@ if (argv.build) {
     }
   );
 
-  const tags = getTags(
-    argv.tag,
-    argv.local,
-    argv.repository,
-    argv.version,
-    argv.name
-  );
-  const tagArgs = tags
-    .map((tag) => ["-t", tag])
-    .reduce((soFar, tagArgs) => soFar.concat(tagArgs), []);
+  // metadata json from GitHub Action docker/metadata-action@v5
+  const metadata =
+    argv.metadata && env.DOCKER_METADATA_OUTPUT_JSON
+      ? JSON.parse(env.DOCKER_METADATA_OUTPUT_JSON)
+      : undefined;
+
+  const tags = metadata
+    ? metadata.tags
+    : getTags(argv.tag, argv.local, argv.repository, argv.version, argv.name);
+  const tagArgs = tags.flatMap((tag) => ["-t", tag]);
+
+  const annotationArgs =
+    metadata?.annotations?.flatMap((annotation) => [
+      "--annotation",
+      annotation
+    ]) ?? [];
 
   const cacheFromArgs = cacheFromImage ? ["--cache-from", cacheFromImage] : [];
 
@@ -186,6 +206,7 @@ if (argv.build) {
       ...(argv.platform ? ["buildx"] : []),
       "build",
       ...tagArgs,
+      ...annotationArgs,
       ...cacheFromArgs,
       ...(argv.noCache ? ["--no-cache"] : []),
       ...(argv.platform ? ["--platform", argv.platform, "--push"] : []),

package.json

@@ -13,7 +13,7 @@
     ]
   },
   "name": "terriajs-map",
-  "version": "0.1.2",
+  "version": "0.2.0-alpha.4",
   "description": "Geospatial catalog explorer based on TerriaJS.",
   "license": "Apache-2.0",
   "engines": {
@@ -96,7 +96,7 @@
   "scripts": {
     "prepare": "husky install",
     "docker-build-local": "node deploy/docker/create-docker-context.js --build --push --tag auto --local",
-    "docker-build-prod": "node deploy/docker/create-docker-context.js --build --push --tag auto --repository=ghcr.io/terriajs",
+    "docker-build-prod": "node deploy/docker/create-docker-context.js --build --push --platform=linux/amd64,linux/arm64",
     "docker-build-ci": "node deploy/docker/create-docker-context.js --build",
     "start": "terriajs-server --config-file serverconfig.json",
     "gulp": "gulp",