feat: add building and publishing multi-platform images (#6208)

* feat(container): add multi platform builds

* feat(container): publish images without the need to pull them locally

* chore: well there is no armd64 architecture

* chore: small fixes

* chore: remove kaniko from supported multi-platform builders

* chore: fix docs and publish for cloudbuilder without kubernetes

* chore: fix dead link

core/src/plugins/container/build.ts

@@ -7,7 +7,7 @@
  */
 
 import { containerHelpers } from "./helpers.js"
-import { ConfigurationError } from "../../exceptions.js"
+import { ConfigurationError, toGardenError } from "../../exceptions.js"
 import type { PrimitiveMap } from "../../config/common.js"
 import split2 from "split2"
 import type { BuildActionHandler } from "../../plugin/action-types.js"
@@ -166,16 +166,39 @@ async function buildContainerLocally({
   }
 
   const cmdOpts = ["build", ...dockerFlags, "--file", dockerfilePath]
-
-  return await containerHelpers.dockerCli({
-    cwd: buildPath,
-    args: [...cmdOpts, buildPath],
-    log,
-    stdout: outputStream,
-    stderr: outputStream,
-    timeout,
-    ctx,
-  })
+  try {
+    return await containerHelpers.dockerCli({
+      cwd: buildPath,
+      args: [...cmdOpts, buildPath],
+      log,
+      stdout: outputStream,
+      stderr: outputStream,
+      timeout,
+      ctx,
+    })
+  } catch (e) {
+    const error = toGardenError(e)
+    if (error.message.includes("docker exporter does not currently support exporting manifest lists")) {
+      throw new ConfigurationError({
+        message: dedent`
+          Your local docker image store does not support loading multi-platform images.
+          If you are using Docker Desktop, you can turn on the experimental containerd image store.
+          Learn more at https://docs.docker.com/go/build-multi-platform/
+        `,
+      })
+    } else if (error.message.includes("Multi-platform build is not supported for the docker driver")) {
+      throw new ConfigurationError({
+        message: dedent`
+          Your local docker daemon does not support building multi-platform images.
+          If you are using Docker Desktop, you can turn on the experimental containerd image store.
+          To build multi-platform images locally with other local docker platforms,
+          you can add a custom buildx builder of type docker-container.
+          Learn more at https://docs.docker.com/go/build-multi-platform/
+        `,
+      })
+    }
+    throw error
+  }
 }
 
 const BUILDKIT_LAYER_REGEX = /^#[0-9]+ \[[^ ]+ +[0-9]+\/[0-9]+\] [^F][^R][^O][^M]/
@@ -237,7 +260,7 @@ export function getDockerBuildFlags(
 ) {
   const args: string[] = []
 
-  const { targetStage, extraFlags, buildArgs } = action.getSpec()
+  const { targetStage, extraFlags, buildArgs, platforms } = action.getSpec()
 
   for (const arg of getDockerBuildArgs(action.versionString(), buildArgs)) {
     args.push("--build-arg", arg)
@@ -246,6 +269,9 @@ export function getDockerBuildFlags(
   if (targetStage) {
     args.push("--target", targetStage)
   }
+  for (const platform of platforms || []) {
+    args.push("--platform", platform)
+  }
 
   args.push(...(extraFlags || []))
   args.push(...(containerProviderConfig.dockerBuildExtraFlags || []))

core/src/plugins/container/cloudbuilder.ts

@@ -42,6 +42,10 @@ const cloudBuilderAvailability = new LRUCache<string, CloudBuilderAvailability>(
 
 // public API
 export const cloudBuilder = {
+  isConfigured(ctx: PluginContext) {
+    const { isCloudBuilderEnabled } = getConfiguration(ctx)
+    return isCloudBuilderEnabled
+  },
   /**
    * @returns false if Cloud Builder is not configured or not available, otherwise it returns the availability (a required parameter for withBuilder)
    */

core/src/plugins/container/config.ts

@@ -1016,6 +1016,7 @@ export interface ContainerBuildActionSpec {
   localId?: string
   publishId?: string
   targetStage?: string
+  platforms?: string[]
 }
 
 export type ContainerBuildActionConfig = BuildActionConfig<"container", ContainerBuildActionSpec>
@@ -1059,6 +1060,10 @@ export const containerCommonBuildSpecKeys = memoize(() => ({
   extraFlags: joi.sparseArray().items(joi.string()).description(deline`
     Specify extra flags to use when building the container image.
     Note that arguments may not be portable across implementations.`),
+  platforms: joi.sparseArray().items(joi.string()).description(dedent`
+    Specify the platforms to build the image for. This is useful when building multi-platform images.
+    The format is \`os/arch\`, e.g. \`linux/amd64\`, \`linux/arm64\`, etc.
+  `),
 }))
 
 export const containerBuildSpecSchema = createSchema({

core/src/plugins/container/container.ts

@@ -51,6 +51,7 @@ import { joi } from "../../config/common.js"
 import { DEFAULT_DEPLOY_TIMEOUT_SEC, gardenEnv } from "../../constants.js"
 import type { ExecBuildConfig } from "../exec/build.js"
 import type { PluginToolSpec } from "../../plugin/tools.js"
+import type { PluginContext } from "../../plugin-context.js"
 
 export const CONTAINER_STATUS_CONCURRENCY_LIMIT = gardenEnv.GARDEN_HARD_CONCURRENCY_LIMIT
 export const CONTAINER_BUILD_CONCURRENCY_LIMIT_LOCAL = 5
@@ -97,6 +98,7 @@ export const configSchema = () =>
     .unknown(false)
 
 export type ContainerProvider = Provider<ContainerProviderConfig>
+export type ContainerPluginContext = PluginContext<ContainerProviderConfig>
 
 export const dockerVersion = "25.0.2"
 export const dockerSpec: PluginToolSpec = {
@@ -221,6 +223,47 @@ export const namespaceCliSpec: PluginToolSpec = {
   ],
 }
 
+export const regctlCliVersion = "0.6.1"
+export const regctlCliSpec: PluginToolSpec = {
+  name: "regctl",
+  version: regctlCliVersion,
+  description: `Regctl CLI v${regctlCliVersion}`,
+  type: "binary",
+  _includeInGardenImage: true,
+  builds: [
+    {
+      platform: "darwin",
+      architecture: "amd64",
+      url: `https://github.com/regclient/regclient/releases/download/v${regctlCliVersion}/regctl-darwin-amd64`,
+      sha256: "916e17019c36ff537555ad9989eb1fcda07403904bc70f808cee9ed9658d4107",
+    },
+    {
+      platform: "darwin",
+      architecture: "arm64",
+      url: `https://github.com/regclient/regclient/releases/download/v${regctlCliVersion}/regctl-darwin-arm64`,
+      sha256: "28833b2f0b42257e703bf75bfab7dd5baeb52d4a6e3ad8e7d33f754b36b8bb07",
+    },
+    {
+      platform: "linux",
+      architecture: "amd64",
+      url: `https://github.com/regclient/regclient/releases/download/v${regctlCliVersion}/regctl-linux-amd64`,
+      sha256: "e541327d14c8e6d3a2e4b0dfd76046425a1816879d4f5951042791435dec82e3",
+    },
+    {
+      platform: "linux",
+      architecture: "arm64",
+      url: `https://github.com/regclient/regclient/releases/download/v${regctlCliVersion}/regctl-linux-arm64`,
+      sha256: "7c3d760925052f7dea4aa26b327e9d88f3ae30fadacc110ae03bd06df3fb696f",
+    },
+    {
+      platform: "windows",
+      architecture: "amd64",
+      url: `https://github.com/regclient/regclient/releases/download/v${regctlCliVersion}/regctl-windows-amd64.exe`,
+      sha256: "44b2d5e79ef457e575d2b09bc1f27500cf90b733651793f4e76e23c9b8fc1803",
+    },
+  ],
+}
+
 // TODO: remove in 0.14. validation should be in the action validation handler.
 export async function configureContainerModule({ log, moduleConfig }: ConfigureModuleParams<ContainerModule>) {
   // validate services
@@ -679,7 +722,7 @@ export const gardenPlugin = () =>
       },
     ],
 
-    tools: [dockerSpec, namespaceCliSpec],
+    tools: [dockerSpec, namespaceCliSpec, regctlCliSpec],
   })
 
 function validateRuntimeCommon(action: Resolved<ContainerRuntimeAction>) {

core/src/plugins/container/helpers.ts

@@ -417,6 +417,49 @@ const helpers = {
     }
   },
 
+  async regctlCli({
+    cwd,
+    args,
+    log,
+    ctx,
+    ignoreError = false,
+    stdout,
+    stderr,
+    timeout,
+  }: {
+    cwd: string
+    args: string[]
+    log: Log
+    ctx: PluginContext<ContainerProviderConfig>
+    ignoreError?: boolean
+    stdout?: Writable
+    stderr?: Writable
+    timeout?: number
+  }) {
+    const regctl = ctx.tools["container.regctl"]
+
+    try {
+      const res = await regctl.spawnAndWait({
+        args,
+        cwd,
+        ignoreError,
+        log,
+        stdout,
+        stderr,
+        timeoutSec: timeout,
+      })
+      return res
+    } catch (err) {
+      if (!(err instanceof GardenError)) {
+        throw err
+      }
+      throw new RuntimeError({
+        message: `Unable to run regctl command "${args.join(" ")}" in ${cwd}: ${err.message}`,
+        wrappedErrors: [err],
+      })
+    }
+  },
+
   moduleHasDockerfile(config: ContainerModuleConfig, version: ModuleVersion): boolean {
     // If we explicitly set a Dockerfile, we take that to mean you want it to be built.
     // If the file turns out to be missing, this will come up in the build handler.

core/src/plugins/container/publish.ts

@@ -19,19 +19,23 @@ export const publishContainerBuild: BuildActionHandler<"publish", ContainerBuild
 }) => {
   const localImageId = action.getOutput("localImageId")
   const remoteImageId = containerHelpers.getPublicImageId(action, tagOverride)
+  const dockerBuildExtraFlags = action.getSpec("extraFlags")
 
-  const taggedImages = [localImageId, remoteImageId]
-  log.info({ msg: `Tagging images ${naturalList(taggedImages)}` })
-  await containerHelpers.dockerCli({
-    cwd: action.getBuildPath(),
-    args: ["tag", ...taggedImages],
-    log,
-    ctx,
-  })
+  // If --push flag is set explicitly, use regctl to copy the image.
+  // This does not require to pull the image locally.
+  if (dockerBuildExtraFlags?.includes("--push")) {
+    const regctlCopyCommand = ["image", "copy", localImageId, remoteImageId]
+    log.info({ msg: `Publishing image ${remoteImageId}` })
+    await containerHelpers.regctlCli({ cwd: action.getBuildPath(), args: regctlCopyCommand, log, ctx })
+  } else {
+    const taggedImages = [localImageId, remoteImageId]
+    log.info({ msg: `Tagging images ${naturalList(taggedImages)}` })
+    await containerHelpers.dockerCli({ cwd: action.getBuildPath(), args: ["tag", ...taggedImages], log, ctx })
 
-  log.info({ msg: `Publishing image ${remoteImageId}...` })
-  // TODO: stream output to log if at debug log level
-  await containerHelpers.dockerCli({ cwd: action.getBuildPath(), args: ["push", remoteImageId], log, ctx })
+    log.info({ msg: `Publishing image ${remoteImageId}...` })
+    // TODO: stream output to log if at debug log level
+    await containerHelpers.dockerCli({ cwd: action.getBuildPath(), args: ["push", remoteImageId], log, ctx })
+  }
 
   return {
     state: "ready",

core/src/plugins/kubernetes/container/build/buildkit.ts

@@ -314,6 +314,10 @@ export function getBuildkitFlags(action: Resolved<ContainerBuildAction>) {
     args.push("--opt", "target=" + spec.targetStage)
   }
 
+  for (const platform of spec.platforms || []) {
+    args.push("--opt", "platform=" + platform)
+  }
+
   args.push(...(spec.extraFlags || []))
 
   return args

core/src/plugins/kubernetes/container/build/kaniko.ts

@@ -97,6 +97,16 @@ export const kanikoBuild: BuildHandler = async (params) => {
   const deploymentImageId = outputs.deploymentImageId
   const dockerfile = spec.dockerfile || defaultDockerfileName
 
+  const platforms = action.getSpec().platforms
+  if (platforms && platforms.length > 1) {
+    throw new ConfigurationError({
+      message: dedent`Failed building ${styles.bold(action.name)}.
+          Kaniko does not support multi-platform builds.
+          Please consider a build method that supports multi-platform builds.
+          See: https://docs.garden.io/other-plugins/container#multi-platform-builds`,
+    })
+  }
+
   let { authSecret } = await ensureUtilDeployment({
     ctx,
     provider,

core/src/plugins/kubernetes/container/publish.ts

@@ -8,41 +8,37 @@
 
 import type { ContainerBuildAction } from "../../container/moduleConfig.js"
 import type { KubernetesPluginContext } from "../config.js"
-import { pullBuild } from "../commands/pull-image.js"
 import type { BuildActionHandler } from "../../../plugin/action-types.js"
 import { containerHelpers } from "../../container/helpers.js"
 import { naturalList } from "../../../util/string.js"
+import { cloudBuilder } from "../../container/cloudbuilder.js"
 
 export const k8sPublishContainerBuild: BuildActionHandler<"publish", ContainerBuildAction> = async (params) => {
   const { ctx, action, log, tagOverride } = params
   const k8sCtx = ctx as KubernetesPluginContext
   const provider = k8sCtx.provider
 
-  const localImageId = action.getOutput("localImageId")
-
-  if (provider.config.buildMode !== "local-docker") {
-    // NOTE: this may contain a custom deploymentRegistry, from the kubernetes provider config
-    const deploymentRegistryImageId = action.getOutput("deploymentImageId")
-
-    // First pull from the deployment registry, then resume standard publish flow.
-    // This does mean we require a local docker as a go-between, but the upside is that we can rely on the user's
-    // standard authentication setup, instead of having to re-implement or account for all the different ways the
-    // user might be authenticating with their registries.
-    // We also generally prefer this because the remote cluster very likely doesn't (and shouldn't) have
-    // privileges to push to production registries.
-    log.info(`Pulling from deployment registry...`)
-    await pullBuild({ ctx: k8sCtx, action, log, localId: localImageId, remoteId: deploymentRegistryImageId })
-  }
+  const cloudBuilderConfigured = cloudBuilder.isConfigured(k8sCtx)
 
+  const localImageId = action.getOutput("localImageId")
+  const deploymentRegistryImageId = action.getOutput("deploymentImageId")
   const remoteImageId = containerHelpers.getPublicImageId(action, tagOverride)
 
-  const taggedImages = [localImageId, remoteImageId]
-  log.info({ msg: `Tagging images ${naturalList(taggedImages)}` })
-  await containerHelpers.dockerCli({ cwd: action.getBuildPath(), args: ["tag", ...taggedImages], log, ctx })
-
-  log.info({ msg: `Publishing image ${remoteImageId}...` })
-  // TODO: stream output to log if at debug log level
-  await containerHelpers.dockerCli({ cwd: action.getBuildPath(), args: ["push", remoteImageId], log, ctx })
+  // For in-cluster building or cloud builder, use regctl to copy the image.
+  // This does not require to pull the image locally.
+  if (provider.config.buildMode !== "local-docker" || cloudBuilderConfigured) {
+    const regctlCopyCommand = ["image", "copy", deploymentRegistryImageId, remoteImageId]
+    log.info({ msg: `Publishing image ${remoteImageId}` })
+    await containerHelpers.regctlCli({ cwd: action.getBuildPath(), args: regctlCopyCommand, log, ctx })
+  } else {
+    const taggedImages = [localImageId, remoteImageId]
+    log.info({ msg: `Tagging images ${naturalList(taggedImages)}` })
+    await containerHelpers.dockerCli({ cwd: action.getBuildPath(), args: ["tag", ...taggedImages], log, ctx })
+
+    log.info({ msg: `Publishing image ${remoteImageId}...` })
+    // TODO: stream output to log if at debug log level
+    await containerHelpers.dockerCli({ cwd: action.getBuildPath(), args: ["push", remoteImageId], log, ctx })
+  }
 
   return {
     state: "ready",

core/test/integ/src/plugins/container/container.ts

@@ -312,4 +312,21 @@ describe("plugins.container", () => {
       expect(args.slice(2, 4)).to.eql(["--build-arg", `GARDEN_ACTION_VERSION=${buildAction.versionString()}`])
     })
   })
+
+  describe("multiPlatformBuilds", () => {
+    it("should include platform flags", async () => {
+      const config = cloneDeep(baseConfig)
+      config.spec.platforms = ["linux/amd64", "linux/arm64"]
+
+      const buildAction = await getTestBuild(config)
+      const resolvedBuild = await garden.resolveAction({
+        action: buildAction,
+        log,
+        graph: await garden.getConfigGraph({ log, emit: false }),
+      })
+
+      const args = getDockerBuildFlags(resolvedBuild, ctx.provider.config)
+      expect(args.slice(-4)).to.eql(["--platform", "linux/amd64", "--platform", "linux/arm64"])
+    })
+  })
 })

core/test/unit/src/verify-ext-tool-binary-hashes.ts

@@ -11,7 +11,7 @@ import { kubectlSpec } from "../../../src/plugins/kubernetes/kubectl.js"
 import { kustomize4Spec, kustomize5Spec } from "../../../src/plugins/kubernetes/kubernetes-type/kustomize.js"
 import { helm3Spec } from "../../../src/plugins/kubernetes/helm/helm-cli.js"
 import { downloadBinariesAndVerifyHashes } from "../../../src/util/testing.js"
-import { dockerSpec, namespaceCliSpec } from "../../../src/plugins/container/container.js"
+import { dockerSpec, namespaceCliSpec, regctlCliSpec } from "../../../src/plugins/container/container.js"
 
 describe("Docker binaries", () => {
   downloadBinariesAndVerifyHashes([dockerSpec])
@@ -21,6 +21,10 @@ describe("NamespaceCLI binaries", () => {
   downloadBinariesAndVerifyHashes([namespaceCliSpec])
 })
 
+describe("regctlCLI binaries", () => {
+  downloadBinariesAndVerifyHashes([regctlCliSpec])
+})
+
 describe("Mutagen binaries", () => {
   downloadBinariesAndVerifyHashes([mutagenCliSpecNative(), mutagenCliSpecLegacy()])
 })