diff --git a/core/package.json b/core/package.json index d8a590e3f8..a78b63999e 100644 --- a/core/package.json +++ b/core/package.json @@ -100,6 +100,7 @@ "linewrap": "^0.2.1", "lodash-es": "^4.17.21", "log-symbols": "^6.0.0", + "lru-cache": "^10.2.0", "micromatch": "^4.0.5", "mimic-function": "^5.0.0", "minimatch": "^7.1.1", @@ -275,4 +276,4 @@ "fsevents": "^2.3.3" }, "gitHead": "b0647221a4d2ff06952bae58000b104215aed922" -} \ No newline at end of file +} diff --git a/core/src/cloud/api.ts b/core/src/cloud/api.ts index 64779c4fe4..f7bfafead1 100644 --- a/core/src/cloud/api.ts +++ b/core/src/cloud/api.ts @@ -827,6 +827,28 @@ export class CloudApi { return secrets } + async registerCloudBuilderBuild(body: { + actionName: string + actionUid: string + coreSessionId: string + }): Promise { + try { + return await this.post(`/cloudbuilder/builds/`, { + body, + }) + } catch (err) { + return { + data: { + version: "v1", + availability: { + available: false, + reason: `Failed to determine Garden Cloud Builder availability: ${extractErrorMessageBodyFromGotError(err) ?? err}`, + }, + }, + } + } + } + async createEphemeralCluster(): Promise { try { const response = await this.post(`/ephemeral-clusters/`) @@ -851,3 +873,31 @@ export class CloudApi { } } } + +// TODO(cloudbuilder): import these from api-types +type V1RegisterCloudBuilderBuildResponse = { + data: { + version: "v1" + availability: CloudBuilderAvailability + } +} +type UnsupportedRegisterCloudBuilderBuildResponse = { + data: { + version: "unsupported" // using unknown here overpowers the compund type + } +} +type RegisterCloudBuilderBuildResponse = + | V1RegisterCloudBuilderBuildResponse + | UnsupportedRegisterCloudBuilderBuildResponse + +type CloudBuilderAvailable = { + available: true + builder: string + token: string + region: "eu" // location of the builder. Currently only eu is supported +} +type CloudBuilderNotAvailable = { + available: false + reason: string +} +export type CloudBuilderAvailability = CloudBuilderAvailable | CloudBuilderNotAvailable diff --git a/core/src/constants.ts b/core/src/constants.ts index 59513c35d9..0909f3100f 100644 --- a/core/src/constants.ts +++ b/core/src/constants.ts @@ -102,4 +102,7 @@ export const gardenEnv = { .default("https://get.garden.io/releases") .asUrlString(), GARDEN_ENABLE_NEW_SYNC: env.get("GARDEN_ENABLE_NEW_SYNC").required(false).default("false").asBool(), + // GARDEN_CLOUD_BUILDER will always override the config; That's why it doesn't have a default. + // FIXME: If the environment variable is not set, asBool returns undefined, unlike the type suggests. That's why we cast to `boolean | undefined`. + GARDEN_CLOUD_BUILDER: env.get("GARDEN_CLOUD_BUILDER").required(false).asBool() as boolean | undefined, } diff --git a/core/src/plugins/container/build.ts b/core/src/plugins/container/build.ts index c9e25767f1..dd71bef521 100644 --- a/core/src/plugins/container/build.ts +++ b/core/src/plugins/container/build.ts @@ -18,6 +18,12 @@ import type { Resolved } from "../../actions/types.js" import dedent from "dedent" import { splitFirst } from "../../util/string.js" import type { ContainerProviderConfig } from "./container.js" +import type { Writable } from "stream" +import type { ActionLog } from "../../logger/log-entry.js" +import type { PluginContext } from "../../plugin-context.js" +import type { SpawnOutput } from "../../util/util.js" +import { cloudbuilder } from "./cloudbuilder.js" +import { styles } from "../../logger/styles.js" export const getContainerBuildStatus: BuildActionHandler<"getStatus", ContainerBuildAction> = async ({ ctx, @@ -39,43 +45,21 @@ export const getContainerBuildStatus: BuildActionHandler<"getStatus", ContainerB export const buildContainer: BuildActionHandler<"build", ContainerBuildAction> = async ({ ctx, action, log }) => { containerHelpers.checkDockerServerVersion(await containerHelpers.getDockerVersion(), log) - const buildPath = action.getBuildPath() - const spec = action.getSpec() + const outputs = action.getOutputs() + const identifier = outputs.localImageId + const hasDockerfile = await containerHelpers.actionHasDockerfile(action) // make sure we can build the thing if (!hasDockerfile) { throw new ConfigurationError({ message: dedent` - Dockerfile not found at ${spec.dockerfile || defaultDockerfileName} for build ${action.name}. + Dockerfile not found at ${action.getSpec().dockerfile || defaultDockerfileName} for build ${action.name}. Please make sure the file exists, and is not excluded by include/exclude fields or .gardenignore files. `, }) } - const outputs = action.getOutputs() - - const identifier = outputs.localImageId - - // build doesn't exist, so we create it - log.info(`Building ${identifier}...`) - - const dockerfilePath = joinWithPosix(action.getBuildPath(), spec.dockerfile) - - const cmdOpts = [ - "build", - "-t", - identifier, - ...getDockerBuildFlags(action, ctx.provider.config), - "--file", - dockerfilePath, - ] - - // if deploymentImageId is different from localImageId, tag the image with deploymentImageId as well. - if (outputs.deploymentImageId && identifier !== outputs.deploymentImageId) { - cmdOpts.push(...["-t", outputs.deploymentImageId]) - } - const logEventContext = { origin: "docker build", level: "verbose" as const, @@ -87,8 +71,68 @@ export const buildContainer: BuildActionHandler<"build", ContainerBuildAction> = ctx.events.emit("log", { timestamp: new Date().toISOString(), msg: line.toString(), ...logEventContext }) }) const timeout = action.getConfig("timeout") - const res = await containerHelpers.dockerCli({ - cwd: action.getBuildPath(), + + let res: SpawnOutput + if (await cloudbuilder.isConfiguredAndAvailable(ctx, action)) { + res = await buildContainerInCloudBuilder({ action, outputStream, timeout, log, ctx }) + } else { + res = await buildContainerLocally({ + action, + outputStream, + timeout, + log, + ctx, + }) + } + + return { + state: "ready", + outputs, + detail: { fresh: true, buildLog: res.all || "", outputs, details: { identifier } }, + } +} + +async function buildContainerLocally({ + action, + outputStream, + timeout, + log, + ctx, + extraDockerOpts = [], +}: { + action: Resolved + outputStream: Writable + timeout: number + log: ActionLog + ctx: PluginContext + extraDockerOpts?: string[] +}) { + const spec = action.getSpec() + const outputs = action.getOutputs() + const buildPath = action.getBuildPath() + + log.info(`Building ${outputs.localImageId}...`) + + const dockerfilePath = joinWithPosix(buildPath, spec.dockerfile) + + const dockerFlags = [...getDockerBuildFlags(action, ctx.provider.config), ...extraDockerOpts] + + // If there already is a --tag flag, another plugin like the Kubernetes plugin already decided how to tag the image. + // In this case, we don't want to add another local tag. + // TODO: it would be nice to find a better way to become aware of the parent plugin's concerns in the container plugin. + if (!dockerFlags.includes("--tag")) { + dockerFlags.push(...["--tag", outputs.localImageId]) + + // if deploymentImageId is different from localImageId, tag the image with deploymentImageId as well. + if (outputs.deploymentImageId && outputs.localImageId !== outputs.deploymentImageId) { + dockerFlags.push(...["--tag", outputs.deploymentImageId]) + } + } + + const cmdOpts = ["build", ...dockerFlags, "--file", dockerfilePath] + + return await containerHelpers.dockerCli({ + cwd: buildPath, args: [...cmdOpts, buildPath], log, stdout: outputStream, @@ -96,12 +140,54 @@ export const buildContainer: BuildActionHandler<"build", ContainerBuildAction> = timeout, ctx, }) +} - return { - state: "ready", - outputs, - detail: { fresh: true, buildLog: res.all || "", outputs, details: { identifier } }, +const BUILDKIT_LAYER_REGEX = /^#[0-9]+ \[[^ ]+ +[0-9]+\/[0-9]+\] [^F][^R][^O][^M]/ +const BUILDKIT_LAYER_CACHED_REGEX = /^#[0-9]+ CACHED/ + +async function buildContainerInCloudBuilder(params: { + action: Resolved + outputStream: Writable + timeout: number + log: ActionLog + ctx: PluginContext +}) { + const cloudbuilderStats = { + totalLayers: 0, + layersCached: 0, } + + // get basic buildkit stats + params.outputStream.on("data", (line: Buffer) => { + const logLine = line.toString() + if (BUILDKIT_LAYER_REGEX.test(logLine)) { + cloudbuilderStats.totalLayers++ + } else if (BUILDKIT_LAYER_CACHED_REGEX.test(logLine)) { + cloudbuilderStats.layersCached++ + } + }) + + const res = await cloudbuilder.withBuilder(params.ctx, params.action, async (builderName) => { + const extraDockerOpts = ["--builder", builderName] + + // we add --push in the Kubernetes local-docker handler when using the Kubernetes plugin with a deploymentRegistry setting. + // If we have --push, no need to --load. + if (!getDockerBuildFlags(params.action, params.ctx.provider.config).includes("--push")) { + // This action makes sure to download the image from the cloud builder, and make it available locally. + extraDockerOpts.push("--load") + } + + return await buildContainerLocally({ ...params, extraDockerOpts }) + }) + + const log = params.ctx.log.createLog({ + name: `build.${params.action.name}`, + }) + log.success( + `${styles.bold("Accelerated by Garden Cloud Builder")} (${cloudbuilderStats.layersCached}/${cloudbuilderStats.totalLayers} layers cached)` + ) + + return res } export function getContainerBuildActionOutputs(action: Resolved): ContainerBuildOutputs { diff --git a/core/src/plugins/container/cloudbuilder.ts b/core/src/plugins/container/cloudbuilder.ts new file mode 100644 index 0000000000..02d669e2da --- /dev/null +++ b/core/src/plugins/container/cloudbuilder.ts @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +import type { PluginContext } from "../../plugin-context.js" +import type { Resolved } from "../../actions/types.js" +import type { ContainerBuildAction } from "./config.js" +import { ChildProcessError, ConfigurationError, InternalError } from "../../exceptions.js" +import type { ContainerProvider } from "./container.js" +import dedent from "dedent" +import { styles } from "../../logger/styles.js" +import type { KubernetesPluginContext } from "../kubernetes/config.js" +import { uuidv4 } from "../../util/random.js" +import fsExtra from "fs-extra" +const { mkdirp, rm } = fsExtra +import { join } from "path" +import { tmpdir } from "node:os" +import type { CloudBuilderAvailability } from "../../cloud/api.js" +import { emitNonRepeatableWarning } from "../../warnings.js" +import { LRUCache } from "lru-cache" +import { getPlatform } from "../../util/arch-platform.js" +import { gardenEnv } from "../../constants.js" + +type CloudBuilderConfiguration = { + isInClusterBuildingConfigured: boolean + isCloudBuilderEnabled: boolean +} + +// TODO: consider if it's useful to make this tunable e.g. via an environment variable. +const cloudBuilderAvailability = new LRUCache({ + max: 1000, + // 5 minutes + ttl: 1000 * 60 * 5, +}) + +// public API +export const cloudbuilder = { + async isConfiguredAndAvailable(ctx: PluginContext, action: Resolved) { + const { isCloudBuilderEnabled } = getConfiguration(ctx) + if (!isCloudBuilderEnabled) { + return false + } + + if (getPlatform() === "windows") { + emitNonRepeatableWarning( + ctx.log, + dedent` + ${styles.bold("Garden Cloud Builder is not available for Windows at the moment.")} + + Please contact our customer support and tell us more if you're interested in Windows support.` + ) + return false + } + + const availability = await getAvailability(ctx, action) + + return availability.available + }, + + async withBuilder( + ctx: PluginContext, + action: Resolved, + performBuild: (builder: string) => Promise + ) { + const cb = await getAvailability(ctx, action) + if (!cb.available) { + throw new InternalError({ + message: `Must call isConfiguredAndAvailable before calling withBuilder.`, + }) + } + + // Docker only accepts builder names that start with a letter + const buildxBuilderName = `cb${uuidv4()}-${cb.builder}` + + // Temp dir needs to be as short as possible, otherwise docker fails to connect + // (ERROR: no valid drivers found: unix socket path "..." is too long) + const stateDir = join(tmpdir(), buildxBuilderName.substring(0, 8)) + await mkdirp(stateDir) + + try { + ctx.log.debug(`Spawning buildx proxy ${buildxBuilderName}`) + const result = await nscCli({ + // See https://namespace.so/docs/cli/docker-buildx-setup + args: ["docker", "buildx", "setup", "--name", buildxBuilderName, "--state", stateDir, "--background"], + ctx, + nscAuthToken: cb.token, + nscRegion: cb.region, + }) + ctx.log.debug( + `buildx proxy setup process for ${buildxBuilderName} exited with code ${result.exitCode}${result.all?.length ? ` (output: ${result.all})` : ""}` + ) + + return await performBuild(buildxBuilderName) + } finally { + ctx.log.debug(`Cleaning up ${buildxBuilderName}`) + await nscCli({ + args: ["docker", "buildx", "cleanup", "--state", stateDir], + ctx, + nscAuthToken: cb.token, + nscRegion: cb.region, + }) + ctx.log.debug(`Removing ${stateDir}...`) + await rm(stateDir, { recursive: true, force: true }) + } + }, +} + +// private helpers + +async function nscCli({ + args, + ctx, + nscAuthToken, + nscRegion, +}: { + args: string[] + ctx: PluginContext + nscAuthToken: string + nscRegion: string +}) { + // env variables for the nsc commands + const env = { + // skip update check + NS_DO_NOT_UPDATE: "true", + // this helps avoiding to interfere with user's own nsc authentication, if they happen to use it + NSC_TOKEN_SPEC: Buffer.from( + JSON.stringify({ + version: "v1", + inline_token: nscAuthToken, + }) + ) + // nsc uses https://pkg.go.dev/encoding/base64#RawStdEncoding (standard base64 encoding without padding characters) + .toString("base64") + .replaceAll("=", ""), + } + + const nsc = ctx.tools["container.namespace-cli"] + + try { + return await nsc.exec({ args: ["--region", nscRegion, ...args], log: ctx.log, env }) + } catch (e: unknown) { + if (e instanceof ChildProcessError) { + // if an error happens here, it's likely a bug + throw InternalError.wrapError(e, "Failed to set up Garden Cloud Builder") + } else { + throw e + } + } +} + +function getConfiguration(ctx: PluginContext): CloudBuilderConfiguration { + let containerProvider: ContainerProvider + let isInClusterBuildingConfigured: boolean + if (ctx.provider.name === "container") { + containerProvider = ctx.provider as ContainerProvider + isInClusterBuildingConfigured = false + } else if (ctx.provider.name.includes("kubernetes")) { + containerProvider = ctx.provider.dependencies.container as ContainerProvider + const config = (ctx as KubernetesPluginContext).provider.config + isInClusterBuildingConfigured = config.buildMode && config.buildMode !== "local-docker" + } else { + throw new InternalError({ + message: `called cloudbuilder.isAvailable in unsupported plugin named ${ctx.provider.name}`, + }) + } + + let isCloudBuilderEnabled = containerProvider.config.gardenCloudBuilder?.enabled || false + + // The env variable GARDEN_CLOUD_BUILDER can be used to override the cloudbuilder.enabled config setting. + // It will be undefined, if the variable is not set and true/false if GARDEN_CLOUD_BUILDER=1 or GARDEN_CLOUD_BUILDER=0. + const overrideFromEnv = gardenEnv.GARDEN_CLOUD_BUILDER + if (overrideFromEnv !== undefined) { + isCloudBuilderEnabled = overrideFromEnv + } + + return { + isInClusterBuildingConfigured, + isCloudBuilderEnabled, + } +} + +async function getAvailability( + ctx: PluginContext, + action: Resolved +): Promise { + const { isInClusterBuildingConfigured } = getConfiguration(ctx) + + // Cache the Cloud Builder availability response from Backend for 5 minutes in LRU cache + const fromCache = cloudBuilderAvailability.get(action.uid) + if (fromCache) { + return fromCache + } + + if (!ctx.cloudApi) { + const fallbackDescription = isInClusterBuildingConfigured + ? `This forces Garden to use the fall-back option to build images within your Kubernetes cluster, as in-cluster building is configured in the Kubernetes provider settings.` + : `This forces Garden to use the fall-back option to build images locally.` + + throw new ConfigurationError({ + message: dedent` + You are not logged in. Run ${styles.command("garden login")} so Garden Cloud Builder can speed up your container builds. + + If you can't log in right now, disable Garden Cloud Builder using the environment variable ${styles.bold("GARDEN_CLOUD_BUILDER=0")}. ${fallbackDescription}`, + }) + } + + const res = await ctx.cloudApi.registerCloudBuilderBuild({ + // TODO: send requested platforms and action version + actionUid: action.uid, + actionName: action.name, + coreSessionId: ctx.sessionId, + }) + + if (res.data.version !== "v1") { + emitNonRepeatableWarning( + ctx.log, + dedent` + ${styles.bold("Update Garden to continue to benefit from Garden Cloud Builder.")} + + Your current Garden version is not supported anymore by Garden Cloud Builder. Please update Garden to the latest version. + + Falling back to ${isInClusterBuildingConfigured ? "in-cluster building" : "building the image locally"}, which may be slower. + + Run ${styles.command("garden self-update")} to update Garden to the latest version.` + ) + const unsupported: CloudBuilderAvailability = { available: false, reason: "Unsupported client version" } + cloudBuilderAvailability.set(action.uid, unsupported) + return unsupported + } + + // availability is supported + const availability = res.data.availability + cloudBuilderAvailability.set(action.uid, availability) + + if (!availability.available) { + emitNonRepeatableWarning( + ctx.log, + dedent` + ${styles.bold("Garden Cloud Builder is not available.")} + + Falling back to ${isInClusterBuildingConfigured ? "in-cluster building" : "building the image locally"}, which may be slower. + + ${styles.italic(`Reason: ${availability.reason}`)}` + ) + } + + return availability +} diff --git a/core/src/plugins/container/container.ts b/core/src/plugins/container/container.ts index d0261c15f7..38503e7490 100644 --- a/core/src/plugins/container/container.ts +++ b/core/src/plugins/container/container.ts @@ -22,7 +22,7 @@ import { containerModuleOutputsSchema, containerModuleSpecSchema, defaultDockerf import { buildContainer, getContainerBuildActionOutputs, getContainerBuildStatus } from "./build.js" import type { ConfigureModuleParams } from "../../plugin/handlers/Module/configure.js" import { dedent, naturalList } from "../../util/string.js" -import type { Provider, GenericProviderConfig } from "../../config/provider.js" +import type { Provider, BaseProviderConfig } from "../../config/provider.js" import { providerConfigBaseSchema } from "../../config/provider.js" import type { GetModuleOutputsParams } from "../../plugin/handlers/Module/get-outputs.js" import type { ConvertModuleParams } from "../../plugin/handlers/Module/convert.js" @@ -47,8 +47,11 @@ import { DEFAULT_DEPLOY_TIMEOUT_SEC } from "../../constants.js" import type { ExecBuildConfig } from "../exec/build.js" import type { PluginToolSpec } from "../../plugin/tools.js" -export interface ContainerProviderConfig extends GenericProviderConfig { +export interface ContainerProviderConfig extends BaseProviderConfig { dockerBuildExtraFlags?: string[] + gardenCloudBuilder?: { + enabled: boolean + } } export const configSchema = () => @@ -59,6 +62,28 @@ export const configSchema = () => Extra flags to pass to the \`docker build\` command. Will extend the \`spec.extraFlags\` specified in each container Build action. `), + // Cloud builder + gardenCloudBuilder: joi + .object() + .optional() + .keys({ + enabled: joi.boolean().default(false).description(dedent` + **Stability: Experimental**. Subject to breaking changes within minor releases. + + Enable Garden Cloud Builder, which can speed up builds significantly using fast machines and extremely fast caching. + + by running \`GARDEN_CLOUD_BUILDER=1 garden build\` you can try Garden Cloud Builder temporarily without any changes to your Garden configuration. + The environment variable \`GARDEN_CLOUD_BUILDER\` can also be used to override this setting, if enabled in the configuration. Set it to \`false\` or \`0\` to temporarily disable Garden Cloud Builder. + + Under the hood, enabling this option means that Garden will install a remote buildx driver on your local Docker daemon, and use that for builds. See also https://docs.docker.com/build/drivers/remote/ + + If service limits are reached, or Garden Cloud Builder is not available, Garden will fall back to building images locally, or it falls back to building in your Kubernetes cluster in case in-cluster building is configured in the Kubernetes provider configuration. + + Please note that when enabling Cloud Builder together with in-cluster building, you need to authenticate to your \`deploymentRegistry\` from the local machine (e.g. by running \`docker login\`). + `), + }).description(dedent` + **Stability: Experimental**. Subject to breaking changes within minor releases. + `), }) .unknown(false) @@ -125,6 +150,68 @@ export const dockerSpec: PluginToolSpec = { ], } +export const namespaceCliVersion = "0.0.354" +export const namespaceCliSpec: PluginToolSpec = { + name: "namespace-cli", + version: dockerVersion, + description: `Namespace.so CLI v${dockerVersion}`, + type: "binary", + _includeInGardenImage: true, + builds: [ + { + platform: "darwin", + architecture: "amd64", + url: `https://get.namespace.so/packages/nsc/v${namespaceCliVersion}/nsc_${namespaceCliVersion}_darwin_amd64.tar.gz`, + sha256: "a091e5f4afeccfffe30231b3528c318bc3201696e09ac3c07adaf283cea42f91", + extract: { + format: "tar", + targetPath: "nsc", + }, + }, + { + platform: "darwin", + architecture: "arm64", + url: `https://get.namespace.so/packages/nsc/v${namespaceCliVersion}/nsc_${namespaceCliVersion}_darwin_arm64.tar.gz`, + sha256: "7641623358ec141c6ab8d243f5f97eab0417338bb1fd490daaf814947c4ed682", + extract: { + format: "tar", + targetPath: "nsc", + }, + }, + { + platform: "linux", + architecture: "amd64", + url: `https://get.namespace.so/packages/nsc/v${namespaceCliVersion}/nsc_${namespaceCliVersion}_linux_amd64.tar.gz`, + sha256: "8d180cf1c3e2f2861c34e89b722d9a5612888e3889d2d7767b02be955e6fc7ef", + extract: { + format: "tar", + targetPath: "nsc", + }, + }, + { + platform: "linux", + architecture: "arm64", + url: `https://get.namespace.so/packages/nsc/v${namespaceCliVersion}/nsc_${namespaceCliVersion}_linux_arm64.tar.gz`, + sha256: "0646fae1d6ca41888cbcac749b04ad303adcb5b2a7eb5260cddad1d7566ba0d6", + extract: { + format: "tar", + targetPath: "nsc", + }, + }, + // No windows support at the moment, only WSL + // { + // platform: "windows", + // architecture: "amd64", + // url: `https://get.namespace.so/packages/nsc/v${namespaceCliVersion}/nsc_${namespaceCliVersion}_${os}_${architecture}.tar.gz`, + // sha256: "25ff5d9dd8ae176dd30fd97b0b99a896d598fa62fca0b7171b45887ad4d3661b", + // extract: { + // format: "zip", + // targetPath: "docker/docker.exe", + // }, + // }, + ], +} + // TODO: remove in 0.14. validation should be in the action validation handler. export async function configureContainerModule({ log, moduleConfig }: ConfigureModuleParams) { // validate services @@ -582,7 +669,7 @@ export const gardenPlugin = () => }, ], - tools: [dockerSpec], + tools: [dockerSpec, namespaceCliSpec], }) function validateRuntimeCommon(action: Resolved) { diff --git a/core/src/plugins/kubernetes/container/build/local.ts b/core/src/plugins/kubernetes/container/build/local.ts index 1070a36e89..cde4c83e44 100644 --- a/core/src/plugins/kubernetes/container/build/local.ts +++ b/core/src/plugins/kubernetes/container/build/local.ts @@ -17,6 +17,7 @@ import { getManifestInspectArgs } from "./common.js" import type { ContainerBuildAction } from "../../../container/moduleConfig.js" import type { BuildActionParams } from "../../../../plugin/action-types.js" import { k8sGetContainerBuildActionOutputs } from "../handlers.js" +import { cloudbuilder } from "../../../container/cloudbuilder.js" export const getLocalBuildStatus: BuildStatusHandler = async (params) => { const { ctx, action, log } = params @@ -67,26 +68,36 @@ export const getLocalBuildStatus: BuildStatusHandler = async (params) => { export const localBuild: BuildHandler = async (params) => { const { ctx, action, log } = params const provider = ctx.provider as KubernetesProvider - const containerProvider = provider.dependencies.container as ContainerProvider const base = params.base || buildContainer - const buildResult = await base!({ ...params, ctx: { ...ctx, provider: containerProvider } }) + const outputs = k8sGetContainerBuildActionOutputs({ provider, action }) + const localId = outputs.localImageId + const remoteId = outputs.deploymentImageId + + const builtByCloudBuilder = await cloudbuilder.isConfiguredAndAvailable(ctx, action) + + // TODO: Kubernetes plugin and container plugin are a little bit twisted; Container plugin has some awareness of Kubernetes, but in this + // case it can't detect that the image needs to be pushed when using remote builder, because it can't get the Kubernetes config from ctx. + const containerProvider: ContainerProvider = + builtByCloudBuilder && provider.config.deploymentRegistry + ? containerProviderWithAdditionalDockerArgs(provider, ["--tag", remoteId, "--push"]) + : // container provider will add --load when using cloud builder automatically, if --push is not present. + provider.dependencies.container + const buildResult = await base({ ...params, ctx: { ...ctx, provider: containerProvider } }) if (!provider.config.deploymentRegistry) { await loadToLocalK8s(params) return buildResult } - const outputs = k8sGetContainerBuildActionOutputs({ provider, action }) - - const localId = outputs.localImageId - const remoteId = outputs.deploymentImageId - const buildPath = action.getBuildPath() + // Cloud Builder already pushes the image. + if (!builtByCloudBuilder) { + log.info({ msg: `→ Pushing image ${remoteId} to remote...` }) - log.info({ msg: `→ Pushing image ${remoteId} to remote...` }) - - await containerHelpers.dockerCli({ cwd: buildPath, args: ["tag", localId, remoteId], log, ctx }) - await containerHelpers.dockerCli({ cwd: buildPath, args: ["push", remoteId], log, ctx }) + const buildPath = action.getBuildPath() + await containerHelpers.dockerCli({ cwd: buildPath, args: ["tag", localId, remoteId], log, ctx }) + await containerHelpers.dockerCli({ cwd: buildPath, args: ["push", remoteId], log, ctx }) + } return buildResult } @@ -106,3 +117,17 @@ export async function loadToLocalK8s(params: BuildActionParams<"build", Containe await loadImageToMicrok8s({ action, imageId: localImageId, log, ctx }) } } + +function containerProviderWithAdditionalDockerArgs( + provider: KubernetesProvider, + additionalDockerArgs: string[] +): ContainerProvider { + const containerProvider = provider.dependencies.container as ContainerProvider + return { + ...containerProvider, + config: { + ...containerProvider.config, + dockerBuildExtraFlags: [...(containerProvider.config.dockerBuildExtraFlags || []), ...additionalDockerArgs], + }, + } +} diff --git a/core/src/plugins/kubernetes/container/extensions.ts b/core/src/plugins/kubernetes/container/extensions.ts index bc1a2cf9f3..f3edff5e84 100644 --- a/core/src/plugins/kubernetes/container/extensions.ts +++ b/core/src/plugins/kubernetes/container/extensions.ts @@ -6,6 +6,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import type { Resolved } from "../../../actions/types.js" import type { DeepPrimitiveMap } from "../../../config/common.js" import type { BuildActionExtension, @@ -13,13 +14,14 @@ import type { RunActionExtension, TestActionExtension, } from "../../../plugin/action-types.js" +import { cloudbuilder } from "../../container/cloudbuilder.js" import type { ContainerBuildAction, ContainerDeployAction, ContainerRunAction, ContainerTestAction, } from "../../container/config.js" -import type { ContainerBuildMode, KubernetesProvider } from "../config.js" +import type { ContainerBuildMode, KubernetesPluginContext, KubernetesProvider } from "../config.js" import { getPortForwardHandler } from "../port-forward.js" import { k8sGetRunResult } from "../run-results.js" import { k8sGetTestResult } from "../test-results.js" @@ -37,6 +39,22 @@ import { k8sGetContainerDeployStatus } from "./status.js" import { k8sContainerGetSyncStatus, k8sContainerStartSync, k8sContainerStopSync } from "./sync.js" import { k8sContainerTest } from "./test.js" +async function getBuildMode({ + ctx, + action, +}: { + ctx: KubernetesPluginContext + action: Resolved +}): Promise { + if (await cloudbuilder.isConfiguredAndAvailable(ctx, action)) { + // Local build mode knows how to build using Cloud Builder + return "local-docker" + } else { + const provider = ctx.provider + return provider.config.buildMode + } +} + export const k8sContainerBuildExtension = (): BuildActionExtension => ({ name: "container", handlers: { @@ -49,19 +67,26 @@ export const k8sContainerBuildExtension = (): BuildActionExtension { - const { ctx } = params + const { ctx, action } = params - const provider = ctx.provider - const handler = buildHandlers[provider.config.buildMode] + const buildMode = await getBuildMode({ + ctx, + action, + }) + const handler = buildHandlers[buildMode] return handler(params) }, getStatus: async (params) => { - const { ctx } = params - const provider = ctx.provider + const { ctx, action } = params + + const buildMode = await getBuildMode({ + ctx, + action, + }) + const handler = buildStatusHandlers[buildMode] - const handler = buildStatusHandlers[provider.config.buildMode] return handler(params) }, diff --git a/core/src/util/ext-tools.ts b/core/src/util/ext-tools.ts index c7c0e49f14..8e41e7bc88 100644 --- a/core/src/util/ext-tools.ts +++ b/core/src/util/ext-tools.ts @@ -8,7 +8,7 @@ import fsExtra from "fs-extra" const { pathExists, createWriteStream, ensureDir, chmod, remove, move, createReadStream } = fsExtra -import { ConfigurationError, InternalError } from "../exceptions.js" +import { InternalError } from "../exceptions.js" import { join, dirname, basename, posix } from "path" import { getArchitecture, getPlatform, isDarwinARM } from "./arch-platform.js" import { hashString, exec } from "./util.js" @@ -49,13 +49,12 @@ export interface SpawnParams extends ExecParams { rawMode?: boolean // Only used if tty = true. See also: https://nodejs.org/api/tty.html#tty_readstream_setrawmode_mode } -export class CliWrapper { - name: string - protected toolPath: string +export abstract class CliWrapper { + public readonly name: string + protected abstract readonly toolPath: string - constructor({ name, path }: { name: string; path: string }) { + constructor({ name }: { name: string }) { this.name = name - this.toolPath = path } async getPath(_: Log) { @@ -195,17 +194,66 @@ export interface PluginTools { * Note: The binary or archive currently needs to be self-contained and work without further installation steps. */ export class PluginTool extends CliWrapper { - type: string - spec: PluginToolSpec - buildSpec: ToolBuildSpec + public readonly type: string + public readonly spec: PluginToolSpec - private versionDirname: string - protected versionPath: string - protected targetSubpath: string + private readonly rootDir: string private chmodDone: boolean constructor(spec: PluginToolSpec) { - super({ name: spec.name, path: "" }) + super({ name: spec.name }) + + this.rootDir = join(toolsPath, spec.name) + this.type = spec.type + this.spec = spec + this.chmodDone = false + } + + override async getPath(log: Log) { + return this.ensurePath(log) + } + + /** + * The full path to the executable tool binary itself. + */ + protected override get toolPath() { + return join(this.versionPath, this.targetSubpath) + } + + /** + * The name of the root directory for this tool version + */ + private get versionDirname() { + return hashString(this.buildSpec.url, 16) + } + + /** + * The full path to the root directory for this tool version + */ + protected get versionPath() { + return join(this.rootDir, this.versionDirname) + } + + /** + * The path to the tool binary relative to `versionPath` + */ + private get targetSubpath() { + const posixPath = this.buildSpec.extract ? this.buildSpec.extract.targetPath : basename(this.buildSpec.url) + + // return path with platform-specific path separators (i.e. '\' on windows) + return join(...posixPath.split(posix.sep)) + } + + /** + * Lazily find build spec; This means that we only throw in case of missing spec for current platform if the tool is actually used. + */ + private _buildSpec: ToolBuildSpec | undefined + private get buildSpec(): ToolBuildSpec { + if (this._buildSpec) { + return this._buildSpec + } + + const spec = this.spec const platform = getPlatform() const architecture = getArchitecture() @@ -223,43 +271,30 @@ export class PluginTool extends CliWrapper { } if (!buildSpec) { - throw new ConfigurationError({ + // if there's no build spec for the platform, that's a bug and the plugin should be aware of that and/or provide tools for all platforms. + throw new InternalError({ message: `Command ${spec.name} doesn't have a spec for this platform/architecture (${platform}-${architecture}${ darwinARM ? "; without emulation: darwin-arm" : "" })`, }) } - this.buildSpec = buildSpec - - this.name = spec.name - this.type = spec.type - this.spec = spec - this.toolPath = join(toolsPath, this.name) - this.versionDirname = hashString(this.buildSpec.url, 16) - this.versionPath = join(this.toolPath, this.versionDirname) - - this.targetSubpath = this.buildSpec.extract ? this.buildSpec.extract.targetPath : basename(this.buildSpec.url) - this.chmodDone = false - } + this._buildSpec = buildSpec - override async getPath(log: Log) { - return this.ensurePath(log) + return buildSpec } async ensurePath(log: Log) { await this.download(log) - const path = join(this.versionPath, ...this.targetSubpath.split(posix.sep)) if (this.spec.type === "binary") { // Make sure the target path is executable if (!this.chmodDone) { - await chmod(path, 0o755) + await chmod(this.toolPath, 0o755) this.chmodDone = true } } - this.toolPath = path - return path + return this.toolPath } protected async download(log: Log) { @@ -268,8 +303,8 @@ export class PluginTool extends CliWrapper { return } - const tmpPath = join(this.toolPath, this.versionDirname + "." + uuidv4().substr(0, 8)) - const targetAbsPath = join(tmpPath, ...this.targetSubpath.split(posix.sep)) + const tmpPath = join(this.versionPath + "." + uuidv4().substr(0, 8)) + const targetAbsPath = join(tmpPath, this.targetSubpath) const downloadLog = log.createLog().info(`Fetching ${this.name} ${this.spec.version}...`) const debug = downloadLog @@ -319,7 +354,7 @@ export class PluginTool extends CliWrapper { const hash = response.pipe(createHash("sha256")) if (!this.buildSpec.extract) { - const targetExecutable = join(tmpPath, ...this.targetSubpath.split(posix.sep)) + const targetExecutable = join(tmpPath, this.targetSubpath) const writeStream = createWriteStream(targetExecutable) await pipeline(response, writeStream) } else { diff --git a/core/src/util/process.ts b/core/src/util/process.ts index 50782d79d9..8cf013a6bf 100644 --- a/core/src/util/process.ts +++ b/core/src/util/process.ts @@ -49,7 +49,8 @@ export function waitForProcessExit({ proc }: { proc: ChildProcess }): Promise { + return new Promise((resolve, reject) => { + proc.on("error", reject) proc.on("exit", () => { resolve() }) diff --git a/core/test/unit/src/plugins/container/build.ts b/core/test/unit/src/plugins/container/build.ts index fda05a6725..ca6b127629 100644 --- a/core/test/unit/src/plugins/container/build.ts +++ b/core/test/unit/src/plugins/container/build.ts @@ -72,12 +72,12 @@ context("build.ts", () => { function getCmdArgs(action: ResolvedBuildAction, any>, buildPath: string) { return [ "build", - "-t", - "some/image", "--build-arg", `GARDEN_MODULE_VERSION=${action.versionString()}`, "--build-arg", `GARDEN_ACTION_VERSION=${action.versionString()}`, + "--tag", + "some/image", "--file", joinWithPosix(action.getBuildPath(), action.getSpec().dockerfile), buildPath, diff --git a/core/test/unit/src/verify-ext-tool-binary-hashes.ts b/core/test/unit/src/verify-ext-tool-binary-hashes.ts index c556abdf21..7e6a016d8c 100755 --- a/core/test/unit/src/verify-ext-tool-binary-hashes.ts +++ b/core/test/unit/src/verify-ext-tool-binary-hashes.ts @@ -11,12 +11,16 @@ import { kubectlSpec } from "../../../src/plugins/kubernetes/kubectl.js" import { kustomizeSpec } from "../../../src/plugins/kubernetes/kubernetes-type/kustomize.js" import { helm3Spec } from "../../../src/plugins/kubernetes/helm/helm-cli.js" import { downloadBinariesAndVerifyHashes } from "../../../src/util/testing.js" -import { dockerSpec } from "../../../src/plugins/container/container.js" +import { dockerSpec, namespaceCliSpec } from "../../../src/plugins/container/container.js" describe("Docker binaries", () => { downloadBinariesAndVerifyHashes([dockerSpec]) }) +describe("NamespaceCLI binaries", () => { + downloadBinariesAndVerifyHashes([namespaceCliSpec]) +}) + describe("Mutagen binaries", () => { downloadBinariesAndVerifyHashes([mutagenCliSpecNative(), mutagenCliSpecLegacy()]) }) diff --git a/docs/reference/providers/container.md b/docs/reference/providers/container.md index 65e0081bc2..a0b3a4196c 100644 --- a/docs/reference/providers/container.md +++ b/docs/reference/providers/container.md @@ -35,6 +35,29 @@ providers: # Extra flags to pass to the `docker build` command. Will extend the `spec.extraFlags` specified in each container # Build action. dockerBuildExtraFlags: + + # **Stability: Experimental**. Subject to breaking changes within minor releases. + gardenCloudBuilder: + # **Stability: Experimental**. Subject to breaking changes within minor releases. + # + # Enable Garden Cloud Builder, which can speed up builds significantly using fast machines and extremely fast + # caching. + # + # by running `GARDEN_CLOUD_BUILDER=1 garden build` you can try Garden Cloud Builder temporarily without any + # changes to your Garden configuration. + # The environment variable `GARDEN_CLOUD_BUILDER` can also be used to override this setting, if enabled in the + # configuration. Set it to `false` or `0` to temporarily disable Garden Cloud Builder. + # + # Under the hood, enabling this option means that Garden will install a remote buildx driver on your local + # Docker daemon, and use that for builds. See also https://docs.docker.com/build/drivers/remote/ + # + # If service limits are reached, or Garden Cloud Builder is not available, Garden will fall back to building + # images locally, or it falls back to building in your Kubernetes cluster in case in-cluster building is + # configured in the Kubernetes provider configuration. + # + # Please note that when enabling Cloud Builder together with in-cluster building, you need to authenticate to + # your `deploymentRegistry` from the local machine (e.g. by running `docker login`). + enabled: false ``` ## Configuration Keys @@ -110,3 +133,34 @@ Extra flags to pass to the `docker build` command. Will extend the `spec.extraFl | --------------- | -------- | | `array[string]` | No | +### `providers[].gardenCloudBuilder` + +[providers](#providers) > gardenCloudBuilder + +**Stability: Experimental**. Subject to breaking changes within minor releases. + +| Type | Required | +| -------- | -------- | +| `object` | No | + +### `providers[].gardenCloudBuilder.enabled` + +[providers](#providers) > [gardenCloudBuilder](#providersgardencloudbuilder) > enabled + +**Stability: Experimental**. Subject to breaking changes within minor releases. + +Enable Garden Cloud Builder, which can speed up builds significantly using fast machines and extremely fast caching. + +by running `GARDEN_CLOUD_BUILDER=1 garden build` you can try Garden Cloud Builder temporarily without any changes to your Garden configuration. +The environment variable `GARDEN_CLOUD_BUILDER` can also be used to override this setting, if enabled in the configuration. Set it to `false` or `0` to temporarily disable Garden Cloud Builder. + +Under the hood, enabling this option means that Garden will install a remote buildx driver on your local Docker daemon, and use that for builds. See also https://docs.docker.com/build/drivers/remote/ + +If service limits are reached, or Garden Cloud Builder is not available, Garden will fall back to building images locally, or it falls back to building in your Kubernetes cluster in case in-cluster building is configured in the Kubernetes provider configuration. + +Please note that when enabling Cloud Builder together with in-cluster building, you need to authenticate to your `deploymentRegistry` from the local machine (e.g. by running `docker login`). + +| Type | Default | Required | +| --------- | ------- | -------- | +| `boolean` | `false` | No | + diff --git a/package-lock.json b/package-lock.json index d0206f968d..9b84995ce7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1282,6 +1282,7 @@ "linewrap": "^0.2.1", "lodash-es": "^4.17.21", "log-symbols": "^6.0.0", + "lru-cache": "^10.2.0", "micromatch": "^4.0.5", "mimic-function": "^5.0.0", "minimatch": "^7.1.1", @@ -4596,13 +4597,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "core/node_modules/glob/node_modules/path-scurry/node_modules/lru-cache": { - "version": "10.0.1", - "license": "ISC", - "engines": { - "node": "14 || >=16.14" - } - }, "core/node_modules/global-agent": { "version": "3.0.0", "license": "BSD-3-Clause", @@ -6339,6 +6333,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "core/node_modules/lru-cache": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", + "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==", + "engines": { + "node": "14 || >=16.14" + } + }, "core/node_modules/md5": { "version": "2.3.0", "dev": true, diff --git a/plugins/pulumi/src/cli.ts b/plugins/pulumi/src/cli.ts index 57d28ed214..490422b7a3 100644 --- a/plugins/pulumi/src/cli.ts +++ b/plugins/pulumi/src/cli.ts @@ -41,8 +41,9 @@ export function pulumi(ctx: PluginContext, provider: PulumiProvider) { } export class GlobalPulumi extends CliWrapper { + protected override toolPath = "pulumi" constructor() { - super({ name: "pulumi", path: "pulumi" }) + super({ name: "pulumi" }) } override async getPath(_: Log) { diff --git a/plugins/terraform/src/cli.ts b/plugins/terraform/src/cli.ts index cb545ed9fa..99de628f62 100644 --- a/plugins/terraform/src/cli.ts +++ b/plugins/terraform/src/cli.ts @@ -34,8 +34,9 @@ export function terraform(ctx: PluginContext, provider: TerraformProvider) { } export class GlobalTerraform extends CliWrapper { + protected override toolPath = "terraform" constructor() { - super({ name: "terraform", path: "terraform" }) + super({ name: "terraform" }) } override async getPath(_: Log) {