From 252b025d43a4c39f9f4a66f6c4b801ba06c5abc8 Mon Sep 17 00:00:00 2001 From: Philipp Garbe Date: Sun, 11 Jul 2021 08:48:25 +0200 Subject: [PATCH] feat(assets): docker images from tar file (#15438) Allows to use an existing tarball for an container image. It loads the image from the tarball instead of building the image from a Dockerfile. Fixes #15419 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-ecr-assets/README.md | 19 +++ packages/@aws-cdk/aws-ecr-assets/lib/index.ts | 1 + .../aws-ecr-assets/lib/tarball-asset.ts | 85 ++++++++++ .../test/demo-tarball/empty.tar | 0 .../aws-ecr-assets/test/tarball-asset.test.ts | 150 ++++++++++++++++++ packages/@aws-cdk/aws-ecs/README.md | 5 +- .../@aws-cdk/aws-ecs/lib/container-image.ts | 24 ++- 7 files changed, 281 insertions(+), 3 deletions(-) create mode 100644 packages/@aws-cdk/aws-ecr-assets/lib/tarball-asset.ts create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/demo-tarball/empty.tar create mode 100644 packages/@aws-cdk/aws-ecr-assets/test/tarball-asset.test.ts diff --git a/packages/@aws-cdk/aws-ecr-assets/README.md b/packages/@aws-cdk/aws-ecr-assets/README.md index 188d0a4ceb96e..2414dc57c084e 100644 --- a/packages/@aws-cdk/aws-ecr-assets/README.md +++ b/packages/@aws-cdk/aws-ecr-assets/README.md @@ -11,6 +11,8 @@ This module allows bundling Docker images as assets. +## Images from Dockerfile + Images are built from a local Docker context directory (with a `Dockerfile`), uploaded to ECR by the CDK toolkit and/or your app's CI-CD pipeline, and can be naturally referenced in your CDK app. @@ -69,6 +71,23 @@ const asset = new DockerImageAsset(this, 'MyBuildImage', { }) ``` +## Images from Tarball + +Images are loaded from a local tarball, uploaded to ECR by the CDK toolkit and/or your app's CI-CD pipeline, and can be +naturally referenced in your CDK app. + +```ts +import { TarballImageAsset } from '@aws-cdk/aws-ecr-assets'; + +const asset = new TarballImageAsset(this, 'MyBuildImage', { + tarballFile: 'local-image.tar' +}); +``` + +This will instruct the toolkit to add the tarball as a file asset. During deployment it will load the container image +from `local-image.tar`, push it to an AWS ECR repository and wire the name of the repository as CloudFormation parameters +to your stack. + ## Publishing images to ECR repositories `DockerImageAsset` is designed for seamless build & consumption of image assets by CDK code deployed to multiple environments diff --git a/packages/@aws-cdk/aws-ecr-assets/lib/index.ts b/packages/@aws-cdk/aws-ecr-assets/lib/index.ts index 579fee533587d..e770bbd197383 100644 --- a/packages/@aws-cdk/aws-ecr-assets/lib/index.ts +++ b/packages/@aws-cdk/aws-ecr-assets/lib/index.ts @@ -1 +1,2 @@ export * from './image-asset'; +export * from './tarball-asset'; diff --git a/packages/@aws-cdk/aws-ecr-assets/lib/tarball-asset.ts b/packages/@aws-cdk/aws-ecr-assets/lib/tarball-asset.ts new file mode 100644 index 0000000000000..48af505e1148e --- /dev/null +++ b/packages/@aws-cdk/aws-ecr-assets/lib/tarball-asset.ts @@ -0,0 +1,85 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import * as ecr from '@aws-cdk/aws-ecr'; +import { AssetStaging, Stack, Stage } from '@aws-cdk/core'; +import { Construct } from 'constructs'; + +// keep this import separate from other imports to reduce chance for merge conflicts with v2-main +// eslint-disable-next-line +import { IAsset } from '@aws-cdk/assets'; +// keep this import separate from other imports to reduce chance for merge conflicts with v2-main +// eslint-disable-next-line no-duplicate-imports, import/order +import { Construct as CoreConstruct } from '@aws-cdk/core'; + +/** + * Options for TarballImageAsset + */ +export interface TarballImageAssetProps { + /** + * Path to the tarball. + */ + readonly tarballFile: string; +} + +/** + * An asset that represents a Docker image. + * + * The image will loaded from an existing tarball and uploaded to an ECR repository. + */ +export class TarballImageAsset extends CoreConstruct implements IAsset { + /** + * The full URI of the image (including a tag). Use this reference to pull + * the asset. + */ + public imageUri: string; + + /** + * Repository where the image is stored + */ + public repository: ecr.IRepository; + + /** + * A hash of the source of this asset, which is available at construction time. As this is a plain + * string, it can be used in construct IDs in order to enforce creation of a new resource when + * the content hash has changed. + * @deprecated use assetHash + */ + public readonly sourceHash: string; + + /** + * A hash of this asset, which is available at construction time. As this is a plain string, it + * can be used in construct IDs in order to enforce creation of a new resource when the content + * hash has changed. + */ + public readonly assetHash: string; + + constructor(scope: Construct, id: string, props: TarballImageAssetProps) { + super(scope, id); + + if (!fs.existsSync(props.tarballFile)) { + throw new Error(`Cannot find file at ${props.tarballFile}`); + } + + const stagedTarball = new AssetStaging(scope, 'Staging', { sourcePath: props.tarballFile }); + + this.sourceHash = stagedTarball.assetHash; + this.assetHash = stagedTarball.assetHash; + + const stage = Stage.of(this); + const relativePathInOutDir = stage ? path.relative(stage.assetOutdir, stagedTarball.absoluteStagedPath) : stagedTarball.absoluteStagedPath; + + const stack = Stack.of(this); + const location = stack.synthesizer.addDockerImageAsset({ + sourceHash: stagedTarball.assetHash, + executable: [ + 'sh', + '-c', + `docker load -i ${relativePathInOutDir} | sed "s/Loaded image: //g"`, + ], + }); + + this.repository = ecr.Repository.fromRepositoryName(this, 'Repository', location.repositoryName); + this.imageUri = location.imageUri; + } +} + diff --git a/packages/@aws-cdk/aws-ecr-assets/test/demo-tarball/empty.tar b/packages/@aws-cdk/aws-ecr-assets/test/demo-tarball/empty.tar new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-ecr-assets/test/tarball-asset.test.ts b/packages/@aws-cdk/aws-ecr-assets/test/tarball-asset.test.ts new file mode 100644 index 0000000000000..c4654fed87044 --- /dev/null +++ b/packages/@aws-cdk/aws-ecr-assets/test/tarball-asset.test.ts @@ -0,0 +1,150 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { expect as ourExpect, haveResource } from '@aws-cdk/assert-internal'; +import * as iam from '@aws-cdk/aws-iam'; +import * as cxschema from '@aws-cdk/cloud-assembly-schema'; +import { App, Stack } from '@aws-cdk/core'; +import * as cxapi from '@aws-cdk/cx-api'; +import { testFutureBehavior } from 'cdk-build-tools/lib/feature-flag'; +import { TarballImageAsset } from '../lib'; + +/* eslint-disable quote-props */ + +const flags = { [cxapi.NEW_STYLE_STACK_SYNTHESIS_CONTEXT]: true }; + +describe('image asset', () => { + testFutureBehavior('test instantiating Asset Image', flags, App, (app) => { + // GIVEN + const stack = new Stack(app); + const assset = new TarballImageAsset(stack, 'Image', { + tarballFile: __dirname + '/demo-tarball/empty.tar', + }); + + // WHEN + const asm = app.synth(); + + // THEN + const manifestArtifact = getAssetManifest(asm); + const manifest = readAssetManifest(manifestArtifact); + + expect(Object.keys(manifest.files ?? {}).length).toBe(1); + expect(Object.keys(manifest.dockerImages ?? {}).length).toBe(1); + + expect(manifest.dockerImages?.[assset.assetHash]?.destinations?.['current_account-current_region']).toStrictEqual( + { + assumeRoleArn: 'arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-image-publishing-role-${AWS::AccountId}-${AWS::Region}', + imageTag: assset.assetHash, + repositoryName: 'cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}', + }, + ); + + expect(manifest.dockerImages?.[assset.assetHash]?.source).toStrictEqual( + { + executable: [ + 'sh', + '-c', + `docker load -i asset.${assset.assetHash}.tar | sed "s/Loaded image: //g"`, + ], + }, + ); + }); + + testFutureBehavior('asset.repository.grantPull can be used to grant a principal permissions to use the image', flags, App, (app) => { + // GIVEN + const stack = new Stack(app); + const user = new iam.User(stack, 'MyUser'); + const asset = new TarballImageAsset(stack, 'Image', { + tarballFile: 'test/demo-tarball/empty.tar', + }); + + // WHEN + asset.repository.grantPull(user); + + // THEN + ourExpect(stack).to(haveResource('AWS::IAM::Policy', { + PolicyDocument: { + 'Statement': [ + { + 'Action': [ + 'ecr:BatchCheckLayerAvailability', + 'ecr:GetDownloadUrlForLayer', + 'ecr:BatchGetImage', + ], + 'Effect': 'Allow', + 'Resource': { + 'Fn::Join': [ + '', + [ + 'arn:', + { + 'Ref': 'AWS::Partition', + }, + ':ecr:', + { + 'Ref': 'AWS::Region', + }, + ':', + { + 'Ref': 'AWS::AccountId', + }, + ':repository/', + { + 'Fn::Sub': 'cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}', + }, + ], + ], + }, + }, + { + 'Action': 'ecr:GetAuthorizationToken', + 'Effect': 'Allow', + 'Resource': '*', + }, + ], + 'Version': '2012-10-17', + }, + 'PolicyName': 'MyUserDefaultPolicy7B897426', + 'Users': [ + { + 'Ref': 'MyUserDC45028B', + }, + ], + })); + }); + + testFutureBehavior('docker directory is staged if asset staging is enabled', flags, App, (app) => { + const stack = new Stack(app); + const image = new TarballImageAsset(stack, 'MyAsset', { + tarballFile: 'test/demo-tarball/empty.tar', + }); + + const session = app.synth(); + + expect(fs.existsSync(path.join(session.directory, `asset.${image.assetHash}.tar`))).toBeDefined(); + }); + + test('fails if the file does not exist', () => { + const stack = new Stack(); + // THEN + expect(() => { + new TarballImageAsset(stack, 'MyAsset', { + tarballFile: `/does/not/exist/${Math.floor(Math.random() * 9999)}`, + }); + }).toThrow(/Cannot find file at/); + + }); +}); + +function isAssetManifest(x: cxapi.CloudArtifact): x is cxapi.AssetManifestArtifact { + return x instanceof cxapi.AssetManifestArtifact; +} + +function getAssetManifest(asm: cxapi.CloudAssembly): cxapi.AssetManifestArtifact { + const manifestArtifact = asm.artifacts.filter(isAssetManifest)[0]; + if (!manifestArtifact) { throw new Error('no asset manifest in assembly'); } + return manifestArtifact; +} + +function readAssetManifest(manifestArtifact: cxapi.AssetManifestArtifact): cxschema.AssetManifest { + return JSON.parse(fs.readFileSync(manifestArtifact.file, { encoding: 'utf-8' })); +} diff --git a/packages/@aws-cdk/aws-ecs/README.md b/packages/@aws-cdk/aws-ecs/README.md index 449d67f986ac4..7edb8e0584370 100644 --- a/packages/@aws-cdk/aws-ecs/README.md +++ b/packages/@aws-cdk/aws-ecs/README.md @@ -302,7 +302,7 @@ const taskDefinition = new ecs.TaskDefinition(this, 'TaskDef', { ### Images Images supply the software that runs inside the container. Images can be -obtained from either DockerHub or from ECR repositories, or built directly from a local Dockerfile. +obtained from either DockerHub or from ECR repositories, built directly from a local Dockerfile, or use an existing tarball. - `ecs.ContainerImage.fromRegistry(imageName)`: use a public image. - `ecs.ContainerImage.fromRegistry(imageName, { credentials: mySecret })`: use a private image that requires credentials. @@ -312,7 +312,8 @@ obtained from either DockerHub or from ECR repositories, or built directly from image directly from a `Dockerfile` in your source directory. - `ecs.ContainerImage.fromDockerImageAsset(asset)`: uses an existing `@aws-cdk/aws-ecr-assets.DockerImageAsset` as a container image. -- `new ecs.TagParameterContainerImage(repository)`: use the given ECR repository as the image +- `ecs.ContainerImage.fromTarball(file)`: use an existing tarball. +- `new ecs.TagParameterContainerImage(repository)`: use the given ECR repository as the image but a CloudFormation parameter as the tag. ### Environment variables diff --git a/packages/@aws-cdk/aws-ecs/lib/container-image.ts b/packages/@aws-cdk/aws-ecs/lib/container-image.ts index 52ca23dcb5aed..05b098fdafedd 100644 --- a/packages/@aws-cdk/aws-ecs/lib/container-image.ts +++ b/packages/@aws-cdk/aws-ecs/lib/container-image.ts @@ -1,4 +1,5 @@ import * as ecr from '@aws-cdk/aws-ecr'; +import { DockerImageAsset, TarballImageAsset } from '@aws-cdk/aws-ecr-assets'; import { ContainerDefinition } from './container-definition'; import { CfnTaskDefinition } from './ecs.generated'; @@ -52,6 +53,28 @@ export abstract class ContainerImage { }; } + /** + * Use an existing tarball for this container image. + * + * Use this method if the container image has already been created by another process (e.g. jib) + * and you want to add it as a container image asset. + * + * @param tarballFile Path to the tarball (relative to the directory). + */ + public static fromTarball(tarballFile: string): ContainerImage { + return { + bind(scope: CoreConstruct, containerDefinition: ContainerDefinition): ContainerImageConfig { + + const asset = new TarballImageAsset(scope, 'Tarball', { tarballFile }); + asset.repository.grantPull(containerDefinition.taskDefinition.obtainExecutionRole()); + + return { + imageName: asset.imageUri, + }; + }, + }; + } + /** * Called when the image is used by a ContainerDefinition */ @@ -73,7 +96,6 @@ export interface ContainerImageConfig { readonly repositoryCredentials?: CfnTaskDefinition.RepositoryCredentialsProperty; } -import { DockerImageAsset } from '@aws-cdk/aws-ecr-assets'; import { AssetImage, AssetImageProps } from './images/asset-image'; import { EcrImage } from './images/ecr'; import { RepositoryImage, RepositoryImageProps } from './images/repository';