From 5607286eb19fa794494b2a7c6c67828099b980a3 Mon Sep 17 00:00:00 2001 From: Otavio Macedo Date: Wed, 27 Jul 2022 12:00:56 +0100 Subject: [PATCH] fix(cli): large context causes E2BIG error during synthesis on Linux (#21230) Instead of passing the context in an environment variable, the CLI now writes the context to a temporary file and sets an environment variable only with the location. The app then uses that location to read from the file. Also tested manually on a Linux machine. Fixes https://github.com/aws/aws-cdk/issues/19261. ### All Submissions: * [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) ### Adding new Unconventional Dependencies: * [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies) ### New Features * [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)? * [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)? *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/core/lib/app.ts | 20 +++++++++---- packages/@aws-cdk/core/test/app.test.ts | 37 +++++++++++++++++++++---- packages/@aws-cdk/cx-api/lib/cxapi.ts | 5 ++++ packages/aws-cdk/lib/api/cxapp/exec.ts | 13 +++++++-- packages/aws-cdk/lib/commands/doctor.ts | 2 +- 5 files changed, 63 insertions(+), 14 deletions(-) diff --git a/packages/@aws-cdk/core/lib/app.ts b/packages/@aws-cdk/core/lib/app.ts index 09ad1a3f81b79..fc535e1620f12 100644 --- a/packages/@aws-cdk/core/lib/app.ts +++ b/packages/@aws-cdk/core/lib/app.ts @@ -1,5 +1,6 @@ import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; +import * as fs from 'fs-extra'; import { addCustomSynthesis, ICustomSynthesis } from './private/synthesis'; import { TreeMetadata } from './private/tree-metadata'; import { Stage } from './stage'; @@ -142,16 +143,23 @@ export class App extends Stage { this.node.setContext(k, v); } - // read from environment - const contextJson = process.env[cxapi.CONTEXT_ENV]; - const contextFromEnvironment = contextJson - ? JSON.parse(contextJson) - : { }; - for (const [k, v] of Object.entries(contextFromEnvironment)) { + const context = this.readContextFromTempFile() ?? this.readContextFromEnvironment() ?? {}; + for (const [k, v] of Object.entries(context)) { this.node.setContext(k, v); } } + + private readContextFromTempFile() { + const location = process.env[cxapi.CONTEXT_LOCATION_ENV]; + return location != null ? fs.readJSONSync(location) : undefined; + } + + // for backward compatibility with old versions of the CLI + private readContextFromEnvironment() { + const contextJson = process.env[cxapi.CONTEXT_ENV]; + return contextJson ? JSON.parse(contextJson) : undefined; + } } /** diff --git a/packages/@aws-cdk/core/test/app.test.ts b/packages/@aws-cdk/core/test/app.test.ts index 0cf612d690f1a..d2ebab512f475 100644 --- a/packages/@aws-cdk/core/test/app.test.ts +++ b/packages/@aws-cdk/core/test/app.test.ts @@ -1,6 +1,9 @@ +import * as os from 'os'; +import * as path from 'path'; import { ContextProvider } from '@aws-cdk/cloud-assembly-schema'; import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; +import * as fs from 'fs-extra'; import { CfnResource, DefaultStackSynthesizer, Stack, StackProps } from '../lib'; import { Annotations } from '../lib/annotations'; import { App, AppProps } from '../lib/app'; @@ -101,25 +104,49 @@ describe('app', () => { }); }); - test('context can be passed through CDK_CONTEXT', () => { - process.env[cxapi.CONTEXT_ENV] = JSON.stringify({ + test('context can be passed through CDK_CONTEXT_LOCATION', async () => { + const contextDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-context')); + const contextLocation = path.join(contextDir, 'context-temp.json'); + fs.writeJSONSync(contextLocation, { key1: 'val1', key2: 'val2', }); + process.env[cxapi.CONTEXT_LOCATION_ENV] = contextLocation; + const prog = new App(); expect(prog.node.tryGetContext('key1')).toEqual('val1'); expect(prog.node.tryGetContext('key2')).toEqual('val2'); }); - test('context passed through CDK_CONTEXT has precedence', () => { + test('context can be passed through CDK_CONTEXT', async () => { process.env[cxapi.CONTEXT_ENV] = JSON.stringify({ key1: 'val1', key2: 'val2', }); + + const prog = new App(); + expect(prog.node.tryGetContext('key1')).toEqual('val1'); + expect(prog.node.tryGetContext('key2')).toEqual('val2'); + }); + + test('context passed through CDK_CONTEXT_LOCATION has precedence', async () => { + const contextDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-context')); + const contextLocation = path.join(contextDir, 'context-temp.json'); + fs.writeJSONSync(contextLocation, { + key1: 'val1', + key2: 'val2', + }); + process.env[cxapi.CONTEXT_LOCATION_ENV] = contextLocation; + + process.env[cxapi.CONTEXT_ENV] = JSON.stringify({ + key1: 'val3', + key2: 'val4', + }); + const prog = new App({ context: { - key1: 'val3', - key2: 'val4', + key1: 'val5', + key2: 'val6', }, }); expect(prog.node.tryGetContext('key1')).toEqual('val1'); diff --git a/packages/@aws-cdk/cx-api/lib/cxapi.ts b/packages/@aws-cdk/cx-api/lib/cxapi.ts index 9b179e9a71b5f..f2e751b3cfad5 100644 --- a/packages/@aws-cdk/cx-api/lib/cxapi.ts +++ b/packages/@aws-cdk/cx-api/lib/cxapi.ts @@ -4,6 +4,11 @@ export const OUTDIR_ENV = 'CDK_OUTDIR'; export const CONTEXT_ENV = 'CDK_CONTEXT_JSON'; +/** + * The name of the temporary file where the context is stored. + */ +export const CONTEXT_LOCATION_ENV = 'CDK_CONTEXT_LOCATION'; + /** * Environment variable set by the CDK CLI with the default AWS account ID. */ diff --git a/packages/aws-cdk/lib/api/cxapp/exec.ts b/packages/aws-cdk/lib/api/cxapp/exec.ts index 34fae3c59c4c6..2f01e3a4a900d 100644 --- a/packages/aws-cdk/lib/api/cxapp/exec.ts +++ b/packages/aws-cdk/lib/api/cxapp/exec.ts @@ -1,4 +1,5 @@ import * as childProcess from 'child_process'; +import * as os from 'os'; import * as path from 'path'; import * as cxschema from '@aws-cdk/cloud-assembly-schema'; import * as cxapi from '@aws-cdk/cx-api'; @@ -44,7 +45,11 @@ export async function execProgram(aws: SdkProvider, config: Configuration): Prom context[cxapi.BUNDLING_STACKS] = bundlingStacks; debug('context:', context); - env[cxapi.CONTEXT_ENV] = JSON.stringify(context); + + const contextDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-context')); + const contextLocation = path.join(contextDir, 'context-temp.json'); + fs.writeJSONSync(contextLocation, context); + env[cxapi.CONTEXT_LOCATION_ENV] = contextLocation; const build = config.settings.get(['build']); if (build) { @@ -85,7 +90,11 @@ export async function execProgram(aws: SdkProvider, config: Configuration): Prom await exec(commandLine.join(' ')); - return createAssembly(outdir); + const assembly = createAssembly(outdir); + + fs.removeSync(path.dirname(contextLocation)); + + return assembly; function createAssembly(appDir: string) { try { diff --git a/packages/aws-cdk/lib/commands/doctor.ts b/packages/aws-cdk/lib/commands/doctor.ts index e1942bbd06b2b..cd8c615341821 100644 --- a/packages/aws-cdk/lib/commands/doctor.ts +++ b/packages/aws-cdk/lib/commands/doctor.ts @@ -51,7 +51,7 @@ function displayCdkEnvironmentVariables() { print('ℹ️ CDK environment variables:'); let healthy = true; for (const key of keys.sort()) { - if (key === cxapi.CONTEXT_ENV || key === cxapi.OUTDIR_ENV) { + if (key === cxapi.CONTEXT_ENV || key === cxapi.CONTEXT_LOCATION_ENV || key === cxapi.OUTDIR_ENV) { print(` - ${chalk.red(key)} = ${chalk.green(process.env[key]!)} (⚠️ reserved for use by the CDK toolkit)`); healthy = false; } else {