diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.assets.json b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.assets.json index c145d851c4c83..f4ff110239f7a 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.assets.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.assets.json @@ -1,7 +1,7 @@ { - "version": "32.0.0", + "version": "36.0.0", "files": { - "fadd6238a0ec388dc8d2cd3046a20a4218f1bde1c230b59050ee6df26f62dc94": { + "8c9962322a7c4c8c90d88a3b9414e1c31d71f87e07f05b6d795e6d8355d5033a": { "source": { "path": "aws-cdk-codepipeline-ecs-deploy.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "fadd6238a0ec388dc8d2cd3046a20a4218f1bde1c230b59050ee6df26f62dc94.json", + "objectKey": "8c9962322a7c4c8c90d88a3b9414e1c31d71f87e07f05b6d795e6d8355d5033a.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.template.json b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.template.json index 67d54e84dc3e0..a2a555c5539b4 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.template.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.template.json @@ -18,9 +18,6 @@ "VPCPublicSubnet1SubnetB4246D30": { "Type": "AWS::EC2::Subnet", "Properties": { - "VpcId": { - "Ref": "VPCB9E5F0B4" - }, "AvailabilityZone": { "Fn::Select": [ 0, @@ -44,21 +41,24 @@ "Key": "Name", "Value": "aws-cdk-codepipeline-ecs-deploy/VPC/PublicSubnet1" } - ] + ], + "VpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "VPCPublicSubnet1RouteTableFEE4B781": { "Type": "AWS::EC2::RouteTable", "Properties": { - "VpcId": { - "Ref": "VPCB9E5F0B4" - }, "Tags": [ { "Key": "Name", "Value": "aws-cdk-codepipeline-ecs-deploy/VPC/PublicSubnet1" } - ] + ], + "VpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "VPCPublicSubnet1RouteTableAssociation0B0896DC": { @@ -75,12 +75,12 @@ "VPCPublicSubnet1DefaultRoute91CEF279": { "Type": "AWS::EC2::Route", "Properties": { - "RouteTableId": { - "Ref": "VPCPublicSubnet1RouteTableFEE4B781" - }, "DestinationCidrBlock": "0.0.0.0/0", "GatewayId": { "Ref": "VPCIGWB7E252D3" + }, + "RouteTableId": { + "Ref": "VPCPublicSubnet1RouteTableFEE4B781" } }, "DependsOn": [ @@ -102,15 +102,15 @@ "VPCPublicSubnet1NATGatewayE0556630": { "Type": "AWS::EC2::NatGateway", "Properties": { - "SubnetId": { - "Ref": "VPCPublicSubnet1SubnetB4246D30" - }, "AllocationId": { "Fn::GetAtt": [ "VPCPublicSubnet1EIP6AD938E8", "AllocationId" ] }, + "SubnetId": { + "Ref": "VPCPublicSubnet1SubnetB4246D30" + }, "Tags": [ { "Key": "Name", @@ -126,9 +126,6 @@ "VPCPrivateSubnet1Subnet8BCA10E0": { "Type": "AWS::EC2::Subnet", "Properties": { - "VpcId": { - "Ref": "VPCB9E5F0B4" - }, "AvailabilityZone": { "Fn::Select": [ 0, @@ -152,21 +149,24 @@ "Key": "Name", "Value": "aws-cdk-codepipeline-ecs-deploy/VPC/PrivateSubnet1" } - ] + ], + "VpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "VPCPrivateSubnet1RouteTableBE8A6027": { "Type": "AWS::EC2::RouteTable", "Properties": { - "VpcId": { - "Ref": "VPCB9E5F0B4" - }, "Tags": [ { "Key": "Name", "Value": "aws-cdk-codepipeline-ecs-deploy/VPC/PrivateSubnet1" } - ] + ], + "VpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "VPCPrivateSubnet1RouteTableAssociation347902D1": { @@ -183,12 +183,12 @@ "VPCPrivateSubnet1DefaultRouteAE1D6490": { "Type": "AWS::EC2::Route", "Properties": { - "RouteTableId": { - "Ref": "VPCPrivateSubnet1RouteTableBE8A6027" - }, "DestinationCidrBlock": "0.0.0.0/0", "NatGatewayId": { "Ref": "VPCPublicSubnet1NATGatewayE0556630" + }, + "RouteTableId": { + "Ref": "VPCPrivateSubnet1RouteTableBE8A6027" } } }, @@ -206,11 +206,11 @@ "VPCVPCGW99B986DC": { "Type": "AWS::EC2::VPCGatewayAttachment", "Properties": { - "VpcId": { - "Ref": "VPCB9E5F0B4" - }, "InternetGatewayId": { "Ref": "VPCIGWB7E252D3" + }, + "VpcId": { + "Ref": "VPCB9E5F0B4" } } }, @@ -492,6 +492,10 @@ "Artifacts": { "Type": "CODEPIPELINE" }, + "Cache": { + "Type": "NO_CACHE" + }, + "EncryptionKey": "alias/aws/s3", "Environment": { "ComputeType": "BUILD_GENERAL1_SMALL", "EnvironmentVariables": [ @@ -562,11 +566,7 @@ "Source": { "BuildSpec": "{\n \"version\": \"0.2\",\n \"phases\": {\n \"pre_build\": {\n \"commands\": \"$(aws ecr get-login --region $AWS_DEFAULT_REGION --no-include-email)\"\n },\n \"build\": {\n \"commands\": \"docker build -t $REPOSITORY_URI:latest .\"\n },\n \"post_build\": {\n \"commands\": [\n \"docker push $REPOSITORY_URI:latest\",\n \"printf '[{ \\\"name\\\": \\\"Container\\\", \\\"imageUri\\\": \\\"%s\\\" }]' $REPOSITORY_URI:latest > imagedefinitions.json\"\n ]\n }\n },\n \"artifacts\": {\n \"files\": \"imagedefinitions.json\"\n }\n}", "Type": "CODEPIPELINE" - }, - "Cache": { - "Type": "NO_CACHE" - }, - "EncryptionKey": "alias/aws/s3" + } } }, "MyPipelineRoleC0D47CA4": { @@ -666,6 +666,12 @@ "MyPipelineAED38ECF": { "Type": "AWS::CodePipeline::Pipeline", "Properties": { + "ArtifactStore": { + "Location": { + "Ref": "MyBucketF68F3FF0" + }, + "Type": "S3" + }, "RoleArn": { "Fn::GetAtt": [ "MyPipelineRoleC0D47CA4", @@ -779,13 +785,7 @@ ], "Name": "Deploy" } - ], - "ArtifactStore": { - "Location": { - "Ref": "MyBucketF68F3FF0" - }, - "Type": "S3" - } + ] }, "DependsOn": [ "MyPipelineRoleDefaultPolicy34F09EFA", @@ -1010,6 +1010,7 @@ "ecs:DescribeTasks", "ecs:ListTasks", "ecs:RegisterTaskDefinition", + "ecs:TagResource", "ecs:UpdateService" ], "Effect": "Allow", diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/cdk.out b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/cdk.out index f0b901e7c06e5..1f0068d32659a 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/cdk.out +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"32.0.0"} \ No newline at end of file +{"version":"36.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/integ.json b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/integ.json index 376ce77f7d56b..b786b10b30c0e 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/integ.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "testCases": { "integ.pipeline-ecs-deploy": { "stacks": [ diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/manifest.json b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/manifest.json index efd6c4aa1e491..846a7917d24c5 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/manifest.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "artifacts": { "aws-cdk-codepipeline-ecs-deploy.assets": { "type": "cdk:asset-manifest", @@ -14,10 +14,11 @@ "environment": "aws://unknown-account/unknown-region", "properties": { "templateFile": "aws-cdk-codepipeline-ecs-deploy.template.json", + "terminationProtection": false, "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/fadd6238a0ec388dc8d2cd3046a20a4218f1bde1c230b59050ee6df26f62dc94.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/8c9962322a7c4c8c90d88a3b9414e1c31d71f87e07f05b6d795e6d8355d5033a.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -171,6 +172,12 @@ "data": "EcsProject54EFDCA6" } ], + "/aws-cdk-codepipeline-ecs-deploy/MyPipeline": [ + { + "type": "aws:cdk:warning", + "data": "V1 pipeline type is implicitly selected when `pipelineType` is not set. If you want to use V2 type, set `PipelineType.V2`. [ack: @aws-cdk/aws-codepipeline:unspecifiedPipelineType]" + } + ], "/aws-cdk-codepipeline-ecs-deploy/MyPipeline/Role/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/tree.json b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/tree.json index 9b1b21b788c29..bcb07e812f369 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/tree.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/tree.json @@ -45,9 +45,6 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::Subnet", "aws:cdk:cloudformation:props": { - "vpcId": { - "Ref": "VPCB9E5F0B4" - }, "availabilityZone": { "Fn::Select": [ 0, @@ -71,7 +68,10 @@ "key": "Name", "value": "aws-cdk-codepipeline-ecs-deploy/VPC/PublicSubnet1" } - ] + ], + "vpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "constructInfo": { @@ -93,15 +93,15 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::RouteTable", "aws:cdk:cloudformation:props": { - "vpcId": { - "Ref": "VPCB9E5F0B4" - }, "tags": [ { "key": "Name", "value": "aws-cdk-codepipeline-ecs-deploy/VPC/PublicSubnet1" } - ] + ], + "vpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "constructInfo": { @@ -134,12 +134,12 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::Route", "aws:cdk:cloudformation:props": { - "routeTableId": { - "Ref": "VPCPublicSubnet1RouteTableFEE4B781" - }, "destinationCidrBlock": "0.0.0.0/0", "gatewayId": { "Ref": "VPCIGWB7E252D3" + }, + "routeTableId": { + "Ref": "VPCPublicSubnet1RouteTableFEE4B781" } } }, @@ -174,15 +174,15 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::NatGateway", "aws:cdk:cloudformation:props": { - "subnetId": { - "Ref": "VPCPublicSubnet1SubnetB4246D30" - }, "allocationId": { "Fn::GetAtt": [ "VPCPublicSubnet1EIP6AD938E8", "AllocationId" ] }, + "subnetId": { + "Ref": "VPCPublicSubnet1SubnetB4246D30" + }, "tags": [ { "key": "Name", @@ -212,9 +212,6 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::Subnet", "aws:cdk:cloudformation:props": { - "vpcId": { - "Ref": "VPCB9E5F0B4" - }, "availabilityZone": { "Fn::Select": [ 0, @@ -238,7 +235,10 @@ "key": "Name", "value": "aws-cdk-codepipeline-ecs-deploy/VPC/PrivateSubnet1" } - ] + ], + "vpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "constructInfo": { @@ -260,15 +260,15 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::RouteTable", "aws:cdk:cloudformation:props": { - "vpcId": { - "Ref": "VPCB9E5F0B4" - }, "tags": [ { "key": "Name", "value": "aws-cdk-codepipeline-ecs-deploy/VPC/PrivateSubnet1" } - ] + ], + "vpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "constructInfo": { @@ -301,12 +301,12 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::Route", "aws:cdk:cloudformation:props": { - "routeTableId": { - "Ref": "VPCPrivateSubnet1RouteTableBE8A6027" - }, "destinationCidrBlock": "0.0.0.0/0", "natGatewayId": { "Ref": "VPCPublicSubnet1NATGatewayE0556630" + }, + "routeTableId": { + "Ref": "VPCPrivateSubnet1RouteTableBE8A6027" } } }, @@ -346,11 +346,11 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::VPCGatewayAttachment", "aws:cdk:cloudformation:props": { - "vpcId": { - "Ref": "VPCB9E5F0B4" - }, "internetGatewayId": { "Ref": "VPCIGWB7E252D3" + }, + "vpcId": { + "Ref": "VPCB9E5F0B4" } } }, @@ -828,6 +828,10 @@ "artifacts": { "type": "CODEPIPELINE" }, + "cache": { + "type": "NO_CACHE" + }, + "encryptionKey": "alias/aws/s3", "environment": { "type": "LINUX_CONTAINER", "image": "aws/codebuild/docker:17.09.0", @@ -898,11 +902,7 @@ "source": { "type": "CODEPIPELINE", "buildSpec": "{\n \"version\": \"0.2\",\n \"phases\": {\n \"pre_build\": {\n \"commands\": \"$(aws ecr get-login --region $AWS_DEFAULT_REGION --no-include-email)\"\n },\n \"build\": {\n \"commands\": \"docker build -t $REPOSITORY_URI:latest .\"\n },\n \"post_build\": {\n \"commands\": [\n \"docker push $REPOSITORY_URI:latest\",\n \"printf '[{ \\\"name\\\": \\\"Container\\\", \\\"imageUri\\\": \\\"%s\\\" }]' $REPOSITORY_URI:latest > imagedefinitions.json\"\n ]\n }\n },\n \"artifacts\": {\n \"files\": \"imagedefinitions.json\"\n }\n}" - }, - "cache": { - "type": "NO_CACHE" - }, - "encryptionKey": "alias/aws/s3" + } } }, "constructInfo": { @@ -1064,6 +1064,12 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::CodePipeline::Pipeline", "aws:cdk:cloudformation:props": { + "artifactStore": { + "type": "S3", + "location": { + "Ref": "MyBucketF68F3FF0" + } + }, "roleArn": { "Fn::GetAtt": [ "MyPipelineRoleC0D47CA4", @@ -1177,13 +1183,7 @@ } ] } - ], - "artifactStore": { - "type": "S3", - "location": { - "Ref": "MyBucketF68F3FF0" - } - } + ] } }, "constructInfo": { @@ -1358,13 +1358,13 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } }, "Build": { @@ -1484,13 +1484,13 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } }, "Deploy": { @@ -1572,6 +1572,7 @@ "ecs:DescribeTasks", "ecs:ListTasks", "ecs:RegisterTaskDefinition", + "ecs:TagResource", "ecs:UpdateService" ], "Effect": "Allow", @@ -1651,13 +1652,13 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, @@ -1693,7 +1694,7 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts index 15cb5757e88b3..4bee0a054bcd8 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts @@ -169,14 +169,32 @@ interface PythonExecutableProps { /** * Additional Python files that AWS Glue adds to the Python path before executing your script. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-py-files`. * * @default - no extra python files and argument is not set * - * @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraPythonFiles?: Code[]; } +interface RayExecutableProps { + /** + * The Python version to use. + */ + readonly pythonVersion: PythonVersion; + + /** + * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--s3-py-modules`. + * + * @default - no extra python files and argument is not set + * + * @see https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + */ + readonly s3PythonModules?: Code[]; +} + interface SharedJobExecutableProps { /** * Runtime. It is required for Ray jobs. @@ -199,10 +217,11 @@ interface SharedJobExecutableProps { /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-files`. * * @default [] - no extra files are copied to the working directory * - * @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraFiles?: Code[]; } @@ -211,19 +230,21 @@ interface SharedSparkJobExecutableProps extends SharedJobExecutableProps { /** * Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-jars`. * * @default [] - no extra jars are added to the classpath * - * @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJars?: Code[]; /** * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * Equivalent to a job parameter `--user-jars-first`. * * @default false - priority is not given to user-provided jars * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJarsFirst?: boolean; } @@ -234,8 +255,9 @@ interface SharedSparkJobExecutableProps extends SharedJobExecutableProps { export interface ScalaJobExecutableProps extends SharedSparkJobExecutableProps { /** * The fully qualified Scala class name that serves as the entry point for the job. + * Equivalent to a job parameter `--class`. * - * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className: string; } @@ -253,7 +275,7 @@ export interface PythonShellExecutableProps extends SharedJobExecutableProps, Py /** * Props for creating a Python Ray job executable */ -export interface PythonRayExecutableProps extends SharedJobExecutableProps, PythonExecutableProps {} +export interface PythonRayExecutableProps extends SharedJobExecutableProps, RayExecutableProps {} /** * The executable properties related to the Glue job's GlueVersion, JobType and code @@ -377,6 +399,9 @@ export class JobExecutable { if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) { throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON'); } + if (config.extraPythonFiles && type === JobType.RAY.name) { + throw new Error('extraPythonFiles is not supported for Ray jobs'); + } if (config.pythonVersion === PythonVersion.THREE_NINE && type !== JobType.PYTHON_SHELL.name && type !== JobType.RAY.name) { throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell and Ray'); } @@ -384,7 +409,7 @@ export class JobExecutable { throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray'); } if (config.runtime === undefined && type === JobType.RAY.name) { - throw new Error('Runtime is required for Ray jobs.'); + throw new Error('Runtime is required for Ray jobs'); } this.config = config; } @@ -410,8 +435,9 @@ export interface JobExecutableConfig { /** * The language of the job (Scala or Python). + * Equivalent to a job parameter `--job-language`. * - * @see `--job-language` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly language: JobLanguage; @@ -441,46 +467,61 @@ export interface JobExecutableConfig { /** * The Scala class that serves as the entry point for the job. This applies only if your the job langauage is Scala. + * Equivalent to a job parameter `--class`. * * @default - no scala className specified * - * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className?: string; /** * Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script. + * Equivalent to a job parameter `--extra-jars`. * * @default - no extra jars specified. * - * @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJars?: Code[]; /** * Additional Python files that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--extra-py-files`. * * @default - no extra python files specified. * - * @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraPythonFiles?: Code[]; + /** + * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--s3-py-modules`. + * + * @default - no extra python files specified. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + */ + readonly s3PythonModules?: Code[]; + /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. + * Equivalent to a job parameter `--extra-files`. * * @default - no extra files specified. * - * @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraFiles?: Code[]; /** * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * Equivalent to a job parameter `--user-jars-first`. * * @default - extra jars are not prioritized. * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJarsFirst?: boolean; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index 526cb774017c3..8420d380bb675 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -598,10 +598,11 @@ export interface JobProps { /** * Enables the collection of metrics for job profiling. + * Equivalent to a job parameter `--enable-metrics`. * * @default - no profiling metrics emitted. * - * @see `--enable-metrics` at https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly enableProfilingMetrics? :boolean; @@ -801,6 +802,9 @@ export class Job extends JobBase { if (config.extraPythonFiles && config.extraPythonFiles.length > 0) { args['--extra-py-files'] = config.extraPythonFiles.map(code => this.codeS3ObjectUrl(code)).join(','); } + if (config.s3PythonModules && config.s3PythonModules.length > 0) { + args['--s3-py-modules'] = config.s3PythonModules.map(code => this.codeS3ObjectUrl(code)).join(','); + } if (config.extraFiles && config.extraFiles.length > 0) { args['--extra-files'] = config.extraFiles.map(code => this.codeS3ObjectUrl(code)).join(','); } @@ -886,8 +890,8 @@ export class Job extends JobBase { } /** - * Create a CloudWatch Metric that's based on Glue Job events - * {@see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types} + * Create a CloudWatch Metric that's based on Glue Job events. + * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types * The metric has namespace = 'AWS/Events', metricName = 'TriggeredRules' and RuleName = rule.ruleName dimension. * * @param rule for use in setting RuleName dimension value diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip new file mode 100644 index 0000000000000..3056cd4db8f27 Binary files /dev/null and b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip differ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py new file mode 100644 index 0000000000000..4fd465ff27636 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py @@ -0,0 +1,4 @@ +import utils + +args = utils.get_job_args(['JOB_NAME', 'JOB_RUN_ID'], ['--s3-py-modules']) +print(args) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index b371535051dc3..4be1daf9e3601 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "files": { "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { "source": { @@ -14,7 +14,33 @@ } } }, - "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a": { + "1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4": { + "source": { + "path": "asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7": { + "source": { + "path": "asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -22,7 +48,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", + "objectKey": "4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index e524ee21d34da..6ed2d571b6a3e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -149,12 +149,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob20ServiceRoleD520CA20", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -187,6 +181,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -338,12 +338,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob20ServiceRole491E0FFF", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -352,6 +346,12 @@ "GlueVersion": "2.0", "Name": "StreamingJob2.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -507,12 +507,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob30ServiceRole8E675579", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -545,6 +539,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -696,12 +696,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob30ServiceRole443B2FDE", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -710,6 +704,12 @@ "GlueVersion": "3.0", "Name": "StreamingJob3.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -865,12 +865,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob40ServiceRoleBDD9998A", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -903,6 +897,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1054,12 +1054,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob40ServiceRole034BDEBD", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1068,6 +1062,12 @@ "GlueVersion": "4.0", "Name": "StreamingJob4.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1181,12 +1181,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "ShellJobServiceRoleCF97BC4B", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1194,6 +1188,12 @@ }, "GlueVersion": "1.0", "Name": "ShellJob", + "Role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, "Tags": { "key": "value" } @@ -1306,12 +1306,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "ShellJob39ServiceRole2F6F3768", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1319,6 +1313,12 @@ }, "GlueVersion": "1.0", "Name": "ShellJob39", + "Role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, "Tags": { "key": "value" } @@ -1427,25 +1427,37 @@ { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + "/9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py" ] ] } }, - "Role": { - "Fn::GetAtt": [ - "RayJobServiceRole51433C3D", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", + "--s3-py-modules": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, "GlueVersion": "4.0", "Name": "RayJob", "NumberOfWorkers": 2, + "Role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1559,12 +1571,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJobWithFLEXServiceRoleBA7C99A5", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python" }, @@ -1572,6 +1578,12 @@ "GlueVersion": "3.0", "Name": "EtlJobWithFLEX", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, "WorkerType": "G.1X" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out index f0b901e7c06e5..1f0068d32659a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"32.0.0"} \ No newline at end of file +{"version":"36.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json index 3e404e817257e..9618d2538fb71 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "testCases": { "integ.job": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index f729d71dca775..e0a0106226e31 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "artifacts": { "aws-glue-job.assets": { "type": "cdk:asset-manifest", @@ -14,10 +14,11 @@ "environment": "aws://unknown-account/unknown-region", "properties": { "templateFile": "aws-glue-job.template.json", + "terminationProtection": false, "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index 1aad2f99aa438..6d88a4cb8fd4a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -20,8 +20,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -59,8 +59,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -165,20 +165,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -193,40 +193,40 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, - "Code161d2d609eea130c2e9c740956bb950c": { - "id": "Code161d2d609eea130c2e9c740956bb950c", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c", + "Code2fe0fc936e45d982e718ad516d9c48b5": { + "id": "Code2fe0fc936e45d982e718ad516d9c48b5", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5", "children": { "Stage": { "id": "Stage", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/Stage", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/Stage", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" } }, "AssetBucket": { "id": "AssetBucket", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/AssetBucket", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/AssetBucket", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" } }, "Resource": { @@ -251,12 +251,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob20ServiceRoleD520CA20", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -289,6 +283,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -297,8 +297,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -346,20 +346,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob2.0": { @@ -374,8 +374,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -413,8 +413,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -482,20 +482,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -520,12 +520,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob20ServiceRole491E0FFF", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -534,6 +528,12 @@ "glueVersion": "2.0", "name": "StreamingJob2.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -541,14 +541,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJob3.0": { @@ -563,8 +563,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -602,8 +602,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -708,20 +708,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -736,14 +736,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, "Resource": { @@ -768,12 +768,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob30ServiceRole8E675579", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -806,6 +800,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -814,8 +814,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -863,20 +863,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob3.0": { @@ -891,8 +891,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -930,8 +930,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -999,20 +999,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1037,12 +1037,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob30ServiceRole443B2FDE", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1051,6 +1045,12 @@ "glueVersion": "3.0", "name": "StreamingJob3.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1058,14 +1058,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJob4.0": { @@ -1080,8 +1080,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1119,8 +1119,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1225,20 +1225,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -1253,14 +1253,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, "Resource": { @@ -1285,12 +1285,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob40ServiceRoleBDD9998A", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -1323,6 +1317,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1331,8 +1331,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -1380,20 +1380,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob4.0": { @@ -1408,8 +1408,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1447,8 +1447,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1516,20 +1516,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1554,12 +1554,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob40ServiceRole034BDEBD", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1568,6 +1562,12 @@ "glueVersion": "4.0", "name": "StreamingJob4.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1575,14 +1575,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "ShellJob": { @@ -1597,8 +1597,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1636,8 +1636,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1705,20 +1705,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1743,12 +1743,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "ShellJobServiceRoleCF97BC4B", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1756,20 +1750,26 @@ }, "glueVersion": "1.0", "name": "ShellJob", + "role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, "tags": { "key": "value" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "ShellJob39": { @@ -1784,8 +1784,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob39/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1823,8 +1823,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1892,20 +1892,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1930,12 +1930,6 @@ }, "pythonVersion": "3.9" }, - "role": { - "Fn::GetAtt": [ - "ShellJob39ServiceRole2F6F3768", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1943,20 +1937,26 @@ }, "glueVersion": "1.0", "name": "ShellJob39", + "role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, "tags": { "key": "value" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "RayJob": { @@ -1971,8 +1971,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/RayJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -2010,8 +2010,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -2079,20 +2079,72 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "Code0115baecff23f24504399384f987872e": { + "id": "Code0115baecff23f24504399384f987872e", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" + } + }, + "Code7e4c8ab0af31dc38a92c1c69bf696244": { + "id": "Code7e4c8ab0af31dc38a92c1c69bf696244", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" } }, "Resource": { @@ -2111,27 +2163,39 @@ { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + "/9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py" ] ] }, "pythonVersion": "3.9", "runtime": "Ray2.4" }, - "role": { - "Fn::GetAtt": [ - "RayJobServiceRole51433C3D", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", + "--s3-py-modules": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, "glueVersion": "4.0", "name": "RayJob", "numberOfWorkers": 2, + "role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -2139,14 +2203,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJobWithFLEX": { @@ -2161,8 +2225,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJobWithFLEX/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -2200,8 +2264,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -2269,20 +2333,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -2307,12 +2371,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJobWithFLEXServiceRoleBA7C99A5", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python" }, @@ -2320,40 +2378,46 @@ "glueVersion": "3.0", "name": "EtlJobWithFLEX", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, "workerType": "G.1X" } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "BootstrapVersion": { "id": "BootstrapVersion", "path": "aws-glue-job/BootstrapVersion", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" } }, "CheckBootstrapVersion": { "id": "CheckBootstrapVersion", "path": "aws-glue-job/CheckBootstrapVersion", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" } }, "Tree": { @@ -2361,13 +2425,13 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.App", + "version": "0.0.0" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts index c16b5f9691ae1..7b405d53ca490 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts @@ -22,6 +22,8 @@ const app = new cdk.App(); const stack = new cdk.Stack(app, 'aws-glue-job'); const script = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_world.py')); +const scriptResolveOptions = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'resolve_options.py')); +const moduleUtils = glue.Code.fromAsset(path.join(__dirname, 'module', 'utils.zip')); [glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { const etlJob = new glue.Job(stack, 'EtlJob' + glueVersion.name, { @@ -113,7 +115,8 @@ new glue.Job(stack, 'RayJob', { glueVersion: glue.GlueVersion.V4_0, pythonVersion: glue.PythonVersion.THREE_NINE, runtime: glue.Runtime.RAY_TWO_FOUR, - script, + s3PythonModules: [moduleUtils], + script: scriptResolveOptions, }), workerType: glue.WorkerType.Z_2X, workerCount: 2, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts index 28c6225c542b6..d00faa55091ba 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts @@ -254,5 +254,29 @@ describe('JobExecutable', () => { script, })).toBeDefined(); }); + + test('with JobTypeof("glueray") and extraPythonFiles set should throw', () => { + expect(() => glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + runtime: glue.Runtime.RAY_TWO_FOUR, + extraPythonFiles: [script], + script, + })).toThrow(/extraPythonFiles is not supported for Ray jobs/); + }); + + test('with JobType ray and s3PythonModules should succeed', () => { + expect(glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + s3PythonModules: [script], + runtime: glue.Runtime.RAY_TWO_FOUR, + script, + })).toBeDefined(); + }); }); }); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py b/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py new file mode 100644 index 0000000000000..4fd465ff27636 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py @@ -0,0 +1,4 @@ +import utils + +args = utils.get_job_args(['JOB_NAME', 'JOB_RUN_ID'], ['--s3-py-modules']) +print(args) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index 748d89b5668a2..48e395a3fedc9 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -871,7 +871,7 @@ describe('Job', () => { }), workerType: glue.WorkerType.Z_2X, workerCount: 2, - })).toThrow('Runtime is required for Ray jobs.'); + })).toThrow('Runtime is required for Ray jobs'); }); }); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip b/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip new file mode 100644 index 0000000000000..3056cd4db8f27 Binary files /dev/null and b/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip differ diff --git a/packages/@aws-cdk/cli-lib-alpha/THIRD_PARTY_LICENSES b/packages/@aws-cdk/cli-lib-alpha/THIRD_PARTY_LICENSES index 1c24888d7bb02..87594dfec653c 100644 --- a/packages/@aws-cdk/cli-lib-alpha/THIRD_PARTY_LICENSES +++ b/packages/@aws-cdk/cli-lib-alpha/THIRD_PARTY_LICENSES @@ -668,7 +668,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI ---------------- -** cdk-from-cfn@0.133.0 - https://www.npmjs.com/package/cdk-from-cfn/v/0.133.0 | MIT OR Apache-2.0 +** cdk-from-cfn@0.141.0 - https://www.npmjs.com/package/cdk-from-cfn/v/0.141.0 | MIT OR Apache-2.0 ---------------- diff --git a/packages/@aws-cdk/cloudformation-diff/lib/diff-template.ts b/packages/@aws-cdk/cloudformation-diff/lib/diff-template.ts index 734753b1d7fc9..1902d757f486d 100644 --- a/packages/@aws-cdk/cloudformation-diff/lib/diff-template.ts +++ b/packages/@aws-cdk/cloudformation-diff/lib/diff-template.ts @@ -37,6 +37,7 @@ const DIFF_HANDLERS: HandlerRegistry = { * @param currentTemplate the current state of the stack. * @param newTemplate the target state of the stack. * @param changeSet the change set for this stack. + * @param isImport if the stack is importing resources (a migrate stack). * * @returns a +types.TemplateDiff+ object that represents the changes that will happen if * a stack which current state is described by +currentTemplate+ is updated with @@ -46,6 +47,7 @@ export function fullDiff( currentTemplate: { [key: string]: any }, newTemplate: { [key: string]: any }, changeSet?: CloudFormation.DescribeChangeSetOutput, + isImport?: boolean, ): types.TemplateDiff { normalize(currentTemplate); @@ -55,6 +57,9 @@ export function fullDiff( filterFalsePositivies(theDiff, changeSet); addImportInformation(theDiff, changeSet); } + if (isImport) { + addImportInformation(theDiff); + } return theDiff; } @@ -209,13 +214,25 @@ function deepCopy(x: any): any { return x; } -function addImportInformation(diff: types.TemplateDiff, changeSet: CloudFormation.DescribeChangeSetOutput) { - const imports = findResourceImports(changeSet); - diff.resources.forEachDifference((logicalId: string, change: types.ResourceDifference) => { - if (imports.includes(logicalId)) { +/** + * Sets import flag to true for resource imports. + * When the changeset parameter is not set, the stack is a new migrate stack, + * so all resource changes are imports. + */ +function addImportInformation(diff: types.TemplateDiff, changeSet?: CloudFormation.DescribeChangeSetOutput) { + if (changeSet) { + const imports = findResourceImports(changeSet); + diff.resources.forEachDifference((logicalId: string, change: types.ResourceDifference) => { + if (imports.includes(logicalId)) { + change.isImport = true; + } + }); + } else { + diff.resources.forEachDifference((logicalId: string, change: types.ResourceDifference) => { + logicalId; // dont know how to get past warning that this variable is not used. change.isImport = true; - } - }); + }); + } } function filterFalsePositivies(diff: types.TemplateDiff, changeSet: CloudFormation.DescribeChangeSetOutput) { diff --git a/packages/aws-cdk-lib/aws-codepipeline-actions/lib/ecs/deploy-action.ts b/packages/aws-cdk-lib/aws-codepipeline-actions/lib/ecs/deploy-action.ts index 2218cc8ce289c..60455d6c18db3 100644 --- a/packages/aws-cdk-lib/aws-codepipeline-actions/lib/ecs/deploy-action.ts +++ b/packages/aws-cdk-lib/aws-codepipeline-actions/lib/ecs/deploy-action.ts @@ -80,7 +80,7 @@ export class EcsDeployAction extends Action { protected bound(_scope: Construct, _stage: codepipeline.IStage, options: codepipeline.ActionBindOptions): codepipeline.ActionConfig { // permissions based on CodePipeline documentation: - // https://docs.aws.amazon.com/codepipeline/latest/userguide/how-to-custom-role.html#how-to-update-role-new-services + // https://docs.aws.amazon.com/codepipeline/latest/userguide/security-iam.html#how-to-custom-role options.role.addToPolicy(new iam.PolicyStatement({ actions: [ 'ecs:DescribeServices', @@ -88,6 +88,7 @@ export class EcsDeployAction extends Action { 'ecs:DescribeTasks', 'ecs:ListTasks', 'ecs:RegisterTaskDefinition', + 'ecs:TagResource', 'ecs:UpdateService', ], resources: ['*'], diff --git a/packages/aws-cdk-lib/aws-elasticloadbalancingv2/lib/nlb/network-target-group.ts b/packages/aws-cdk-lib/aws-elasticloadbalancingv2/lib/nlb/network-target-group.ts index 07b651cac8382..e38ff2e1516bd 100644 --- a/packages/aws-cdk-lib/aws-elasticloadbalancingv2/lib/nlb/network-target-group.ts +++ b/packages/aws-cdk-lib/aws-elasticloadbalancingv2/lib/nlb/network-target-group.ts @@ -280,9 +280,6 @@ export class NetworkTargetGroup extends TargetGroupBase implements INetworkTarge if (timeoutSeconds < lowHealthCheckTimeout || timeoutSeconds > highHealthCheckTimeout) { ret.push(`Health check timeout '${timeoutSeconds}' not supported. Must be a number between ${lowHealthCheckTimeout} and ${highHealthCheckTimeout}.`); } - if (healthCheck.interval && healthCheck.interval.toSeconds() < timeoutSeconds) { - ret.push(`Health check timeout '${timeoutSeconds}' must not be greater than the interval '${healthCheck.interval.toSeconds()}'`); - } } return ret; diff --git a/packages/aws-cdk-lib/aws-elasticloadbalancingv2/lib/shared/base-target-group.ts b/packages/aws-cdk-lib/aws-elasticloadbalancingv2/lib/shared/base-target-group.ts index 024f6303336a7..9e6fe34b24fc7 100644 --- a/packages/aws-cdk-lib/aws-elasticloadbalancingv2/lib/shared/base-target-group.ts +++ b/packages/aws-cdk-lib/aws-elasticloadbalancingv2/lib/shared/base-target-group.ts @@ -280,6 +280,7 @@ export abstract class TargetGroupBase extends Construct implements ITargetGroup this.targetGroupName = this.resource.attrTargetGroupName; this.defaultPort = additionalProps.port; + this.node.addValidation({ validate: () => this.validateHealthCheck() }); this.node.addValidation({ validate: () => this.validateTargetGroup() }); } @@ -351,6 +352,22 @@ export abstract class TargetGroupBase extends Construct implements ITargetGroup return ret; } + + protected validateHealthCheck(): string[] { + const ret = new Array(); + + const intervalSeconds = this.healthCheck.interval?.toSeconds(); + const timeoutSeconds = this.healthCheck.timeout?.toSeconds(); + + if (intervalSeconds && timeoutSeconds) { + if (intervalSeconds < timeoutSeconds) { + // < instead of <= for backwards compatibility, see discussion in https://github.com/aws/aws-cdk/pull/26031 + ret.push('Health check interval must be greater than or equal to the timeout; received interval ' + + `${intervalSeconds}, timeout ${timeoutSeconds}.`); + } + } + return ret; + } } /** diff --git a/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/alb/target-group.test.ts b/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/alb/target-group.test.ts index 60928abdc46ae..7e30c3db10ecb 100644 --- a/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/alb/target-group.test.ts +++ b/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/alb/target-group.test.ts @@ -549,6 +549,45 @@ describe('tests', () => { }).toThrow('Healthcheck interval 1 minute must be greater than the timeout 2 minute'); }); + test('Throws error for health check interval less than timeout', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const vpc = new ec2.Vpc(stack, 'Vpc'); + + new elbv2.ApplicationTargetGroup(stack, 'TargetGroup', { + vpc, + port: 80, + healthCheck: { + interval: cdk.Duration.seconds(10), + timeout: cdk.Duration.seconds(20), + }, + }); + + expect(() => { + app.synth(); + }).toThrow('Health check interval must be greater than or equal to the timeout; received interval 10, timeout 20.'); + }); + + // for backwards compatibility these can be equal, see discussion in https://github.com/aws/aws-cdk/pull/26031 + test('Throws error for health check interval less than timeout', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const vpc = new ec2.Vpc(stack, 'Vpc'); + + new elbv2.ApplicationTargetGroup(stack, 'TargetGroup', { + vpc, + port: 80, + healthCheck: { + interval: cdk.Duration.seconds(10), + timeout: cdk.Duration.seconds(20), + }, + }); + + expect(() => { + app.synth(); + }).toThrow('Health check interval must be greater than or equal to the timeout; received interval 10, timeout 20.'); + }); + test('imported targetGroup has targetGroupName', () => { // GIVEN const app = new cdk.App(); diff --git a/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/nlb/listener.test.ts b/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/nlb/listener.test.ts index 825ca15c40d2a..f5f520aa6a921 100644 --- a/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/nlb/listener.test.ts +++ b/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/nlb/listener.test.ts @@ -408,7 +408,7 @@ describe('tests', () => { // THEN const validationErrors: string[] = targetGroup.node.validate(); expect(validationErrors).toEqual([ - "Health check timeout '40' must not be greater than the interval '30'", + 'Health check interval must be greater than or equal to the timeout; received interval 30, timeout 40.', ]); }); diff --git a/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/nlb/target-group.test.ts b/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/nlb/target-group.test.ts index 2bff7359e74bf..786b13b6c9302 100644 --- a/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/nlb/target-group.test.ts +++ b/packages/aws-cdk-lib/aws-elasticloadbalancingv2/test/nlb/target-group.test.ts @@ -144,6 +144,45 @@ describe('tests', () => { }).toThrow(/Health check interval '3' not supported. Must be between 5 and 300./); }); + test('Throws error for health check interval less than timeout', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const vpc = new ec2.Vpc(stack, 'Vpc'); + + new elbv2.NetworkTargetGroup(stack, 'Group', { + vpc, + port: 80, + healthCheck: { + interval: cdk.Duration.seconds(10), + timeout: cdk.Duration.seconds(20), + }, + }); + + expect(() => { + app.synth(); + }).toThrow('Health check interval must be greater than or equal to the timeout; received interval 10, timeout 20.'); + }); + + // for backwards compatibility these can be equal, see discussion in https://github.com/aws/aws-cdk/pull/26031 + test('No error for health check interval == timeout', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const vpc = new ec2.Vpc(stack, 'Vpc'); + + new elbv2.NetworkTargetGroup(stack, 'Group', { + vpc, + port: 80, + healthCheck: { + interval: cdk.Duration.seconds(10), + timeout: cdk.Duration.seconds(10), + }, + }); + + expect(() => { + app.synth(); + }).not.toThrow(); + }); + test('targetGroupName unallowed: more than 32 characters', () => { // GIVEN const app = new cdk.App(); diff --git a/packages/aws-cdk-lib/aws-lambda-event-sources/test/s3.test.ts b/packages/aws-cdk-lib/aws-lambda-event-sources/test/s3.test.ts index 67b63190e315f..f5ab8a7fc4bb3 100644 --- a/packages/aws-cdk-lib/aws-lambda-event-sources/test/s3.test.ts +++ b/packages/aws-cdk-lib/aws-lambda-event-sources/test/s3.test.ts @@ -228,86 +228,4 @@ describe('S3EventSource', () => { }, }); }); - test('Cross account buckect access', () => { - // GIVEN - const app = new cdk.App(); - const stack = new cdk.Stack(app, 'stack'); - const fn = new TestFunction(stack, 'Fn'); - - let accountB = '1234567'; - //WHEN - const foreignBucket = - s3.Bucket.fromBucketAttributes(stack, 'ImportedBucket', { - bucketArn: 'arn:aws:s3:::some-bucket-not-in-this-account', - // The account the bucket really lives in - account: accountB, - }); - - // This will generate the IAM bindings - fn.addEventSource(new sources.S3EventSource(foreignBucket as s3.Bucket, - { events: [s3.EventType.OBJECT_CREATED] })); - - // THEN - Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Permission', { - 'Principal': 's3.amazonaws.com', - 'SourceAccount': '1234567', - 'SourceArn': 'arn:aws:s3:::some-bucket-not-in-this-account', - }); - }); - - test('Test bucket account is referenced intrinsicly', () => { - // GIVEN - const stack = new cdk.Stack(); - const fn = new TestFunction(stack, 'Fn'); - const bucket = new s3.Bucket(stack, 'B'); - - // WHEN - fn.addEventSource(new sources.S3EventSource(bucket, { - events: [s3.EventType.OBJECT_CREATED, s3.EventType.OBJECT_REMOVED], - filters: [ - { prefix: 'prefix/' }, - { suffix: '.png' }, - ], - })); - - // THEN - Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Permission', { - 'Principal': 's3.amazonaws.com', - 'SourceAccount': { - 'Ref': 'AWS::AccountId', - }, - 'SourceArn': { - 'Fn::GetAtt': ['B08E7C7AF', 'Arn'], - }, - }); - }); - - test('Default to stack account if bucket account doesnt exist', () => { - // GIVEN - const app = new cdk.App(); - const stack = new cdk.Stack(app, 'stack'); - const fn = new TestFunction(stack, 'Fn'); - - let accountB = ''; - //WHEN - const foreignBucket = - s3.Bucket.fromBucketAttributes(stack, 'ImportedBucket', { - bucketArn: 'arn:aws:s3:::some-bucket-not-in-this-account', - // The account the bucket really lives in - account: accountB, - }); - - // This will generate the IAM bindings - fn.addEventSource(new sources.S3EventSource(foreignBucket as s3.Bucket, - { events: [s3.EventType.OBJECT_CREATED] })); - - // THEN - Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Permission', { - 'Principal': 's3.amazonaws.com', - 'SourceAccount': { - 'Ref': 'AWS::AccountId', - }, - 'SourceArn': 'arn:aws:s3:::some-bucket-not-in-this-account', - }); - }); }); diff --git a/packages/aws-cdk-lib/aws-lambda/lib/event-source-mapping.ts b/packages/aws-cdk-lib/aws-lambda/lib/event-source-mapping.ts index 43548757380ea..1f49011ca1bd5 100644 --- a/packages/aws-cdk-lib/aws-lambda/lib/event-source-mapping.ts +++ b/packages/aws-cdk-lib/aws-lambda/lib/event-source-mapping.ts @@ -292,8 +292,15 @@ export interface IEventSourceMapping extends cdk.IResource { * Usually, you won't need to define the mapping yourself. This will usually be done by * event sources. For example, to add an SQS event source to a function: * - * import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'; - * lambda.addEventSource(new SqsEventSource(sqs)); + * ```ts + * import * as sqs from 'aws-cdk-lib/aws-sqs'; + * import * as eventsources from 'aws-cdk-lib/aws-lambda-event-sources'; + * + * declare const handler: lambda.Function; + * declare const queue: sqs.Queue; + * + * handler.addEventSource(new eventsources.SqsEventSource(queue)); + * ``` * * The `SqsEventSource` class will automatically create the mapping, and will also * modify the Lambda's execution role so it can consume messages from the queue. diff --git a/packages/aws-cdk-lib/aws-rds/lib/proxy.ts b/packages/aws-cdk-lib/aws-rds/lib/proxy.ts index 581b8782db91b..cf076a9668802 100644 --- a/packages/aws-cdk-lib/aws-rds/lib/proxy.ts +++ b/packages/aws-cdk-lib/aws-rds/lib/proxy.ts @@ -104,7 +104,8 @@ export class ProxyTarget { const engineFamily = engine.engineFamily; if (!engineFamily) { - throw new Error(`Engine '${engineDescription(engine)}' does not support proxies`); + throw new Error('RDS proxies require an engine family to be specified on the database cluster or instance. ' + + `No family specified for engine '${engineDescription(engine)}'`); } // allow connecting to the Cluster/Instance from the Proxy diff --git a/packages/aws-cdk-lib/aws-rds/test/proxy.test.ts b/packages/aws-cdk-lib/aws-rds/test/proxy.test.ts index 39d372533a51a..970f13b09908d 100644 --- a/packages/aws-cdk-lib/aws-rds/test/proxy.test.ts +++ b/packages/aws-cdk-lib/aws-rds/test/proxy.test.ts @@ -203,7 +203,7 @@ describe('proxy', () => { vpc, secrets: [new secretsmanager.Secret(stack, 'Secret')], }); - }).toThrow(/Engine 'mariadb-10\.0\.24' does not support proxies/); + }).toThrow(/RDS proxies require an engine family to be specified on the database cluster or instance. No family specified for engine 'mariadb-10\.0\.24'/); }); test('correctly creates a proxy for an imported Cluster if its engine is known', () => { diff --git a/packages/aws-cdk-lib/aws-s3-notifications/lib/lambda.ts b/packages/aws-cdk-lib/aws-s3-notifications/lib/lambda.ts index d7e9758eeecb8..e6159ff8c22aa 100644 --- a/packages/aws-cdk-lib/aws-s3-notifications/lib/lambda.ts +++ b/packages/aws-cdk-lib/aws-s3-notifications/lib/lambda.ts @@ -21,7 +21,7 @@ export class LambdaDestination implements s3.IBucketNotificationDestination { if (bucket.node.tryFindChild(permissionId) === undefined) { this.fn.addPermission(permissionId, { - sourceAccount: bucket.account || Stack.of(bucket).account, + sourceAccount: Stack.of(bucket).account, principal: new iam.ServicePrincipal('s3.amazonaws.com'), sourceArn: bucket.bucketArn, // Placing the permissions node in the same scope as the s3 bucket. diff --git a/packages/aws-cdk-lib/aws-s3/lib/bucket.ts b/packages/aws-cdk-lib/aws-s3/lib/bucket.ts index d1d1c255f3513..a799062afc692 100644 --- a/packages/aws-cdk-lib/aws-s3/lib/bucket.ts +++ b/packages/aws-cdk-lib/aws-s3/lib/bucket.ts @@ -95,11 +95,6 @@ export interface IBucket extends IResource { */ policy?: BucketPolicy; - /** - * The account bucket belongs to. - */ - readonly account?: string; - /** * Adds a statement to the resource policy for a principal (i.e. * account/role/service) to perform actions on this bucket and/or its @@ -1759,7 +1754,6 @@ export class Bucket extends BucketBase { public readonly bucketWebsiteNewUrlFormat = attrs.bucketWebsiteNewUrlFormat ?? false; public readonly encryptionKey = attrs.encryptionKey; public readonly isWebsite = attrs.isWebsite ?? false; - public readonly account = attrs.account; public policy?: BucketPolicy = undefined; protected autoCreatePolicy = false; protected disallowPublicAccess = false; @@ -1973,11 +1967,11 @@ export class Bucket extends BucketBase { if (props.serverAccessLogsBucket instanceof Bucket) { props.serverAccessLogsBucket.allowLogDelivery(this, props.serverAccessLogsPrefix); - // It is possible that `serverAccessLogsBucket` was specified but is some other `IBucket` - // that cannot have the ACLs or bucket policy applied. In that scenario, we should only - // setup log delivery permissions to `this` if a bucket was not specified at all, as documented. - // For example, we should not allow log delivery to `this` if given an imported bucket or - // another situation that causes `instanceof` to fail + // It is possible that `serverAccessLogsBucket` was specified but is some other `IBucket` + // that cannot have the ACLs or bucket policy applied. In that scenario, we should only + // setup log delivery permissions to `this` if a bucket was not specified at all, as documented. + // For example, we should not allow log delivery to `this` if given an imported bucket or + // another situation that causes `instanceof` to fail } else if (!props.serverAccessLogsBucket && props.serverAccessLogsPrefix) { this.allowLogDelivery(this, props.serverAccessLogsPrefix); } else if (props.serverAccessLogsBucket) { @@ -2231,7 +2225,7 @@ export class Bucket extends BucketBase { function parseLifecycleRule(rule: LifecycleRule): CfnBucket.RuleProperty { const enabled = rule.enabled ?? true; if ((rule.expiredObjectDeleteMarker) - && (rule.expiration || rule.expirationDate || self.parseTagFilters(rule.tagFilters))) { + && (rule.expiration || rule.expirationDate || self.parseTagFilters(rule.tagFilters))) { // ExpiredObjectDeleteMarker cannot be specified with ExpirationInDays, ExpirationDate, or TagFilters. throw new Error('ExpiredObjectDeleteMarker cannot be specified with expiration, ExpirationDate, or TagFilters.'); } @@ -2356,7 +2350,7 @@ export class Bucket extends BucketBase { } if (accessControlRequiresObjectOwnership && this.objectOwnership === ObjectOwnership.BUCKET_OWNER_ENFORCED) { - throw new Error(`objectOwnership must be set to "${ObjectOwnership.OBJECT_WRITER}" when accessControl is "${this.accessControl}"`); + throw new Error (`objectOwnership must be set to "${ObjectOwnership.OBJECT_WRITER}" when accessControl is "${this.accessControl}"`); } return { diff --git a/packages/aws-cdk-lib/aws-xray/test/xray.test.ts b/packages/aws-cdk-lib/aws-xray/test/xray.test.ts new file mode 100644 index 0000000000000..6f4df4fc6f0dd --- /dev/null +++ b/packages/aws-cdk-lib/aws-xray/test/xray.test.ts @@ -0,0 +1,41 @@ +import { Match, Template } from '../../assertions'; +import * as iam from '../../aws-iam'; +import * as kms from '../../aws-kms'; +import { CfnParameter, Duration, Stack, App, Token, Tags } from '../../core'; +import * as xray from '../lib'; + +/* eslint-disable quote-props */ + +test('able to add tags to XRay CfnGroup', () => { + const stack = new Stack(); + new xray.CfnGroup(stack, 'Group', { + groupName: 'GroupName', + tags: [{ + key: 'Key', + value: 'Value', + }], + }); + + Template.fromStack(stack).hasResourceProperties('AWS::XRay::Group', { + Tags: [{ + Key: 'Key', + Value: 'Value', + }], + }); +}); + +test('able to add tags through Tags.of()... to XRay CfnGroup', () => { + const stack = new Stack(); + new xray.CfnGroup(stack, 'Group', { + groupName: 'GroupName', + }); + + Tags.of(stack).add('Key', 'Value'); + + Template.fromStack(stack).hasResourceProperties('AWS::XRay::Group', { + Tags: [{ + Key: 'Key', + Value: 'Value', + }], + }); +}); diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index 42c3f9f6ec1f3..2b06a12c6b2d4 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -128,7 +128,7 @@ $ cdk synth $ # Synthesize cloud assembly for StackName, but don't include dependencies $ cdk synth MyStackName --exclusively -$ # Synthesize cloud assembly for StackName, but don't cloudFormation template output to STDOUT +$ # Synthesize cloud assembly for StackName, but don't write CloudFormation template output to STDOUT $ cdk synth MyStackName --quiet ``` diff --git a/packages/aws-cdk/THIRD_PARTY_LICENSES b/packages/aws-cdk/THIRD_PARTY_LICENSES index c947720e91891..a8bdfa43f5001 100644 --- a/packages/aws-cdk/THIRD_PARTY_LICENSES +++ b/packages/aws-cdk/THIRD_PARTY_LICENSES @@ -461,7 +461,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI ---------------- -** cdk-from-cfn@0.133.0 - https://www.npmjs.com/package/cdk-from-cfn/v/0.133.0 | MIT OR Apache-2.0 +** cdk-from-cfn@0.141.0 - https://www.npmjs.com/package/cdk-from-cfn/v/0.141.0 | MIT OR Apache-2.0 ---------------- diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 3e721bb47e32a..c67b02dbc9cf2 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -136,7 +136,14 @@ export class CdkToolkit { throw new Error(`There is no file at ${options.templatePath}`); } - const changeSet = options.changeSet ? await createDiffChangeSet({ + const stackExistsOptions = { + stack: stacks.firstStack, + deployName: stacks.firstStack.stackName, + }; + + const stackExists = await this.props.deployments.stackExists(stackExistsOptions); + + const changeSet = (stackExists && options.changeSet) ? await createDiffChangeSet({ stack: stacks.firstStack, uuid: uuid.v4(), willExecute: false, @@ -168,13 +175,23 @@ export class CdkToolkit { removeNonImportResources(stack); } - const changeSet = options.changeSet ? await createDiffChangeSet({ + const stackExistsOptions = { + stack, + deployName: stack.stackName, + }; + + const stackExists = await this.props.deployments.stackExists(stackExistsOptions); + + // if the stack does not already exist, do not do a changeset + // this prevents race conditions between deleting the dummy changeset stack and deploying the real changeset stack + // migrate stacks that import resources will not previously exist and default to old diff logic + const changeSet = (stackExists && options.changeSet) ? await createDiffChangeSet({ stack, uuid: uuid.v4(), deployments: this.props.deployments, willExecute: false, sdkProvider: this.props.sdkProvider, - parameters: Object.assign({}, parameterMap['*'], parameterMap[stacks.firstStack.stackName]), + parameters: Object.assign({}, parameterMap['*'], parameterMap[stacks.firstStack.stackName]), // should this be stack? resourcesToImport, stream, }) : undefined; @@ -183,10 +200,11 @@ export class CdkToolkit { stream.write('Parameters and rules created during migration do not affect resource configuration.\n'); } + // pass a boolean to print if the stack is a migrate stack in order to set all resource diffs to import const stackCount = options.securityOnly ? (numberFromBool(printSecurityDiff(currentTemplate, stack, RequireApproval.Broadening, changeSet)) > 0 ? 1 : 0) - : (printStackDiff(currentTemplate, stack, strict, contextLines, quiet, changeSet, stream) > 0 ? 1 : 0); + : (printStackDiff(currentTemplate, stack, strict, contextLines, quiet, changeSet, stream, !!resourcesToImport) > 0 ? 1 : 0); diffs += stackCount + nestedStackCount; } diff --git a/packages/aws-cdk/lib/commands/migrate.ts b/packages/aws-cdk/lib/commands/migrate.ts index 1703a60392848..8a3b462877755 100644 --- a/packages/aws-cdk/lib/commands/migrate.ts +++ b/packages/aws-cdk/lib/commands/migrate.ts @@ -82,11 +82,14 @@ export async function generateCdkApp(stackName: string, stack: string, language: * @returns A string representation of a CDK stack file */ export function generateStack(template: string, stackName: string, language: string) { + const formattedStackName = `${camelCase(decamelize(stackName), { pascalCase: true })}Stack`; try { - const formattedStackName = `${camelCase(decamelize(stackName), { pascalCase: true })}Stack`; return cdk_from_cfn.transmute(template, language, formattedStackName); } catch (e) { - throw new Error(`stack generation failed due to error '${(e as Error).message}'`); + const errorMessage = (e as Error).message === 'unreachable' + ? 'template and/or language inputs caused the source code to panic' + : (e as Error).message.replace('TransmuteError: ', ''); + throw new Error(`${formattedStackName} could not be generated because ${errorMessage}`); } } diff --git a/packages/aws-cdk/lib/diff.ts b/packages/aws-cdk/lib/diff.ts index 399f08abeac6a..fa897430ad37b 100644 --- a/packages/aws-cdk/lib/diff.ts +++ b/packages/aws-cdk/lib/diff.ts @@ -23,9 +23,10 @@ export function printStackDiff( context: number, quiet: boolean, changeSet?: CloudFormation.DescribeChangeSetOutput, - stream?: cfnDiff.FormatStream): number { + stream?: cfnDiff.FormatStream, + isImport?: boolean): number { - let diff = cfnDiff.fullDiff(oldTemplate, newTemplate.template, changeSet); + let diff = cfnDiff.fullDiff(oldTemplate, newTemplate.template, changeSet, isImport); // detect and filter out mangled characters from the diff let filteredChangesCount = 0; diff --git a/packages/aws-cdk/package.json b/packages/aws-cdk/package.json index 9778f4452c8e0..fa51f646e6918 100644 --- a/packages/aws-cdk/package.json +++ b/packages/aws-cdk/package.json @@ -105,7 +105,7 @@ "aws-sdk": "^2.1562.0", "camelcase": "^6.3.0", "cdk-assets": "0.0.0", - "cdk-from-cfn": "^0.133.0", + "cdk-from-cfn": "^0.141.0", "chalk": "^4", "chokidar": "^3.6.0", "decamelize": "^5.0.1", diff --git a/packages/aws-cdk/test/cdk-toolkit.test.ts b/packages/aws-cdk/test/cdk-toolkit.test.ts index 1a6f55ef63c52..7b17498d4087a 100644 --- a/packages/aws-cdk/test/cdk-toolkit.test.ts +++ b/packages/aws-cdk/test/cdk-toolkit.test.ts @@ -918,8 +918,8 @@ describe('synth', () => { stackName: 'cannot-generate-template', fromPath: path.join(__dirname, 'commands', 'test-resources', 'templates', 'sqs-template.json'), language: 'rust', - })).rejects.toThrowError('stack generation failed due to error \'unreachable\''); - expect(stderrMock.mock.calls[1][0]).toContain(' ❌ Migrate failed for `cannot-generate-template`: stack generation failed due to error \'unreachable\''); + })).rejects.toThrowError('CannotGenerateTemplateStack could not be generated because template and/or language inputs caused the source code to panic'); + expect(stderrMock.mock.calls[1][0]).toContain(' ❌ Migrate failed for `cannot-generate-template`: CannotGenerateTemplateStack could not be generated because template and/or language inputs caused the source code to panic'); }); cliTest('migrate succeeds for valid template from local path when no lanugage is provided', async (workDir) => { diff --git a/packages/aws-cdk/test/commands/migrate.test.ts b/packages/aws-cdk/test/commands/migrate.test.ts index ed0cef6bd1b70..86dcce27f25bf 100644 --- a/packages/aws-cdk/test/commands/migrate.test.ts +++ b/packages/aws-cdk/test/commands/migrate.test.ts @@ -21,7 +21,9 @@ describe('Migrate Function Tests', () => { const validTemplatePath = path.join(...templatePath, 's3-template.json'); const emptyTemplatePath = path.join(...templatePath, 'empty-template.yml'); + const invalidTemplatePath = path.join(...templatePath, 'rds-template.json'); const validTemplate = readFromPath(validTemplatePath)!; + const invalidTemplate = readFromPath(invalidTemplatePath)!; beforeEach(async () => { sdkProvider = new MockSdkProvider(); @@ -132,7 +134,11 @@ describe('Migrate Function Tests', () => { }); test('generateStack throws error when called for other language', () => { - expect(() => generateStack(validTemplate, 'BadBadBad', 'php')).toThrowError('stack generation failed due to error \'unreachable\''); + expect(() => generateStack(validTemplate, 'BadBadBad', 'php')).toThrowError('BadBadBadStack could not be generated because template and/or language inputs caused the source code to panic'); + }); + + test('generateStack throws error for invalid resource property', () => { + expect(() => generateStack(invalidTemplate, 'VeryBad', 'typescript')).toThrow('VeryBadStack could not be generated because ReadEndpoint is not a valid property for resource RDSCluster of type AWS::RDS::DBCluster'); }); cliTest('generateCdkApp generates the expected cdk app when called for typescript', async (workDir) => { diff --git a/packages/aws-cdk/test/commands/test-resources/templates/rds-template.json b/packages/aws-cdk/test/commands/test-resources/templates/rds-template.json new file mode 100644 index 0000000000000..9348b1b7a9e84 --- /dev/null +++ b/packages/aws-cdk/test/commands/test-resources/templates/rds-template.json @@ -0,0 +1,18 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "RDSCluster": { + "Type": "AWS::RDS::DBCluster", + "Properties": { + "DBClusterIdentifier" : "aurora-postgresql-cluster", + "Engine" : "aurora-postgresql", + "EngineVersion" : "10.7", + "DBClusterParameterGroupName" : "default.aurora-postgresql10", + "EnableCloudwatchLogsExports" : ["postgresql"], + "ReadEndpoint": { + "Address": "http://127.0.0.1:8080" + } + } + } + } +} diff --git a/packages/aws-cdk/test/diff.test.ts b/packages/aws-cdk/test/diff.test.ts index a92c6d53551c3..a7b5905e12f87 100644 --- a/packages/aws-cdk/test/diff.test.ts +++ b/packages/aws-cdk/test/diff.test.ts @@ -12,100 +12,6 @@ let cloudExecutable: MockCloudExecutable; let cloudFormation: jest.Mocked; let toolkit: CdkToolkit; -describe('imports', () => { - beforeEach(() => { - jest.spyOn(cfn, 'createDiffChangeSet').mockImplementation(async () => { - return { - Changes: [ - { - ResourceChange: { - Action: 'Import', - LogicalResourceId: 'Queue', - }, - }, - { - ResourceChange: { - Action: 'Import', - LogicalResourceId: 'Bucket', - }, - }, - { - ResourceChange: { - Action: 'Import', - LogicalResourceId: 'Queue2', - }, - }, - ], - }; - }); - cloudExecutable = new MockCloudExecutable({ - stacks: [{ - stackName: 'A', - template: { - Resources: { - Queue: { - Type: 'AWS::SQS::Queue', - }, - Queue2: { - Type: 'AWS::SQS::Queue', - }, - Bucket: { - Type: 'AWS::S3::Bucket', - }, - }, - }, - }], - }); - - cloudFormation = instanceMockFrom(Deployments); - - toolkit = new CdkToolkit({ - cloudExecutable, - deployments: cloudFormation, - configuration: cloudExecutable.configuration, - sdkProvider: cloudExecutable.sdkProvider, - }); - - // Default implementations - cloudFormation.readCurrentTemplateWithNestedStacks.mockImplementation((_stackArtifact: CloudFormationStackArtifact) => { - return Promise.resolve({ - deployedTemplate: {}, - nestedStackCount: 0, - }); - }); - cloudFormation.deployStack.mockImplementation((options) => Promise.resolve({ - noOp: true, - outputs: {}, - stackArn: '', - stackArtifact: options.stack, - })); - }); - - test('imports', async () => { - // GIVEN - const buffer = new StringWritable(); - - // WHEN - const exitCode = await toolkit.diff({ - stackNames: ['A'], - stream: buffer, - changeSet: true, - }); - - // THEN - const plainTextOutput = buffer.data.replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, ''); - expect(plainTextOutput).toContain(`Stack A -Resources -[←] AWS::SQS::Queue Queue import -[←] AWS::SQS::Queue Queue2 import -[←] AWS::S3::Bucket Bucket import -`); - - expect(buffer.data.trim()).toContain('✨ Number of stacks with differences: 1'); - expect(exitCode).toBe(0); - }); -}); - describe('non-nested stacks', () => { beforeEach(() => { cloudExecutable = new MockCloudExecutable({ diff --git a/tools/@aws-cdk/spec2cdk/lib/cdk/resource-decider.ts b/tools/@aws-cdk/spec2cdk/lib/cdk/resource-decider.ts index 2db62bb3d0b4a..7b981cacb8fd7 100644 --- a/tools/@aws-cdk/spec2cdk/lib/cdk/resource-decider.ts +++ b/tools/@aws-cdk/spec2cdk/lib/cdk/resource-decider.ts @@ -179,7 +179,7 @@ export class ResourceDecider { private handleTagPropertyModern(cfnName: string, prop: Property, variant: TagVariant) { const originalName = propertyNameFromCloudFormation(cfnName); - const originalType = this.converter.typeFromProperty(prop); + const originalType = this.converter.typeFromPropertyForModernTags(prop); this.propsProperties.push({ propertySpec: { diff --git a/tools/@aws-cdk/spec2cdk/lib/cdk/type-converter.ts b/tools/@aws-cdk/spec2cdk/lib/cdk/type-converter.ts index df3a215e503e5..5accddfe15da2 100644 --- a/tools/@aws-cdk/spec2cdk/lib/cdk/type-converter.ts +++ b/tools/@aws-cdk/spec2cdk/lib/cdk/type-converter.ts @@ -96,6 +96,16 @@ export class TypeConverter { return this.typeFromSpecType(this.typeHistoryFromProperty(property)[0]); } + /** + * Return the appropriate typewriter type for a servicespec type for modern tags + * Unlike typeFromProperty, we want to default to use the newest type instead. + */ + public typeFromPropertyForModernTags(property: Property): Type { + // For backwards compatibility reasons we always have to use the original type + const types = this.typeHistoryFromProperty(property); + return this.typeFromSpecType(types[types.length - 1]); + } + /** * Return the full type history for a servicespec property */ diff --git a/yarn.lock b/yarn.lock index 2cbafb6831dea..fd141c7631a27 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6307,10 +6307,10 @@ case@1.6.3, case@^1.6.3: resolved "https://registry.npmjs.org/case/-/case-1.6.3.tgz#0a4386e3e9825351ca2e6216c60467ff5f1ea1c9" integrity sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ== -cdk-from-cfn@^0.133.0: - version "0.133.0" - resolved "https://registry.npmjs.org/cdk-from-cfn/-/cdk-from-cfn-0.133.0.tgz#a18dd2c505c8fc0b7f4947f6d6afb3e7fea5b391" - integrity sha512-Yj0kE+GixlSGLNTodCaZEcIeyxzNQonZAykmsSP7wRSm3yYNfg/2XX1tRWi9hzUGMwqYuQr5GsCBymukU0GKsA== +cdk-from-cfn@^0.141.0: + version "0.141.0" + resolved "https://registry.npmjs.org/cdk-from-cfn/-/cdk-from-cfn-0.141.0.tgz#f480ee87658f8056a933e4684198b050cb783e8f" + integrity sha512-saEHn8rxR3a2b0MpOAjnNMeLOoCKo9lSKYMm7GsAPlzyMOiTmTQeIwKhUAgt8ouVbt4VZPddBukzmfh5+xtCYw== cdk-generate-synthetic-examples@^0.1.304: version "0.1.304"