diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index b07770c6666f7..331c0d8b0b431 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -27,6 +27,8 @@ The `glue.JobExecutable` allows you to specify the type of job, the language to `glue.Code` allows you to refer to the different code assets required by the job, either from an existing S3 location or from a local file path. +`glue.ExecutionClass` allows you to specify `FLEX` or `STANDARD`. `FLEX` is appropriate for non-urgent jobs such as pre-production jobs, testing, and one-time data loads. + ### Spark Jobs These jobs run in an Apache Spark environment managed by AWS Glue. diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index beace10bd8acc..dd6cc425e23e0 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -5,7 +5,7 @@ import * as logs from 'aws-cdk-lib/aws-logs'; import * as s3 from 'aws-cdk-lib/aws-s3'; import * as cdk from 'aws-cdk-lib/core'; import * as constructs from 'constructs'; -import { Code, JobExecutable, JobExecutableConfig, JobType } from '.'; +import { Code, GlueVersion, JobExecutable, JobExecutableConfig, JobType } from '.'; import { IConnection } from './connection'; import { CfnJob } from 'aws-cdk-lib/aws-glue'; import { ISecurityConfiguration } from './security-configuration'; @@ -129,6 +129,26 @@ export enum MetricType { COUNT = 'count', } +/** + * The ExecutionClass whether the job is run with a standard or flexible execution class. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html#aws-glue-api-jobs-job-Job + * @see https://docs.aws.amazon.com/glue/latest/dg/add-job.html + */ +export enum ExecutionClass { + /** + * The flexible execution class is appropriate for time-insensitive jobs whose start + * and completion times may vary. + */ + FLEX = 'FLEX', + + /** + * The standard execution class is ideal for time-sensitive workloads that require fast job + * startup and dedicated resources. + */ + STANDARD = 'STANDARD', +} + /** * Interface representing a created or an imported `Job`. */ @@ -600,6 +620,16 @@ export interface JobProps { * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly continuousLogging?: ContinuousLoggingProps, + + /** + * The ExecutionClass whether the job is run with a standard or flexible execution class. + * + * @default - STANDARD + * + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html#aws-glue-api-jobs-job-Job + * @see https://docs.aws.amazon.com/glue/latest/dg/add-job.html + */ + readonly executionClass?: ExecutionClass, } /** @@ -677,6 +707,18 @@ export class Job extends JobBase { ...this.checkNoReservedArgs(props.defaultArguments), }; + if (props.executionClass === ExecutionClass.FLEX) { + if (executable.type !== JobType.ETL) { + throw new Error('FLEX ExecutionClass is only available for JobType.ETL jobs'); + } + if ([GlueVersion.V0_9, GlueVersion.V1_0, GlueVersion.V2_0].includes(executable.glueVersion)) { + throw new Error('FLEX ExecutionClass is only available for GlueVersion 3.0 or later'); + } + if (props.workerType && (props.workerType !== WorkerType.G_1X && props.workerType !== WorkerType.G_2X)) { + throw new Error('FLEX ExecutionClass is only available for WorkerType G_1X or G_2X'); + } + } + const jobResource = new CfnJob(this, 'Resource', { name: props.jobName, description: props.description, @@ -692,6 +734,7 @@ export class Job extends JobBase { numberOfWorkers: props.workerCount, maxCapacity: props.maxCapacity, maxRetries: props.maxRetries, + executionClass: props.executionClass, executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined, notificationProperty: props.notifyDelayAfter ? { notifyDelayAfter: props.notifyDelayAfter.toMinutes() } : undefined, timeout: props.timeout?.toMinutes(), diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index ae93bf2c5f576..b371535051dc3 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -14,7 +14,7 @@ } } }, - "e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89": { + "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89.json", + "objectKey": "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index 92ffc1c36ba4a..e524ee21d34da 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -176,6 +176,7 @@ "arg2": "value2", "--conf": "valueConf" }, + "ExecutionClass": "STANDARD", "ExecutionProperty": { "MaxConcurrentRuns": 2 }, @@ -533,6 +534,7 @@ "arg2": "value2", "--conf": "valueConf" }, + "ExecutionClass": "STANDARD", "ExecutionProperty": { "MaxConcurrentRuns": 2 }, @@ -890,6 +892,7 @@ "arg2": "value2", "--conf": "valueConf" }, + "ExecutionClass": "STANDARD", "ExecutionProperty": { "MaxConcurrentRuns": 2 }, @@ -1448,6 +1451,129 @@ }, "WorkerType": "Z.2X" } + }, + "EtlJobWithFLEXServiceRoleBA7C99A5": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "EtlJobWithFLEXServiceRoleDefaultPolicyEA63C5FE": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "EtlJobWithFLEXServiceRoleDefaultPolicyEA63C5FE", + "Roles": [ + { + "Ref": "EtlJobWithFLEXServiceRoleBA7C99A5" + } + ] + } + }, + "EtlJobWithFLEX928B4212": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "glueetl", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, + "DefaultArguments": { + "--job-language": "python" + }, + "ExecutionClass": "FLEX", + "GlueVersion": "3.0", + "Name": "EtlJobWithFLEX", + "NumberOfWorkers": 10, + "WorkerType": "G.1X" + } } }, "Parameters": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index 17c822d1ad95a..f729d71dca775 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -231,6 +231,24 @@ "data": "RayJob2F7864D9" } ], + "/aws-glue-job/EtlJobWithFLEX/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJobWithFLEXServiceRoleBA7C99A5" + } + ], + "/aws-glue-job/EtlJobWithFLEX/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJobWithFLEXServiceRoleDefaultPolicyEA63C5FE" + } + ], + "/aws-glue-job/EtlJobWithFLEX/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJobWithFLEX928B4212" + } + ], "/aws-glue-job/BootstrapVersion": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index 3a1205674b7ac..1aad2f99aa438 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -20,8 +20,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -59,8 +59,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "DefaultPolicy": { @@ -165,20 +165,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "SparkUIBucket": { @@ -193,40 +193,40 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.CfnBucket", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.Bucket", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, - "Code2fe0fc936e45d982e718ad516d9c48b5": { - "id": "Code2fe0fc936e45d982e718ad516d9c48b5", - "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5", + "Code161d2d609eea130c2e9c740956bb950c": { + "id": "Code161d2d609eea130c2e9c740956bb950c", + "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c", "children": { "Stage": { "id": "Stage", - "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/Stage", + "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/Stage", "constructInfo": { - "fqn": "aws-cdk-lib.AssetStaging", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "AssetBucket": { "id": "AssetBucket", - "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/AssetBucket", + "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/AssetBucket", "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.BucketBase", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3_assets.Asset", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -278,6 +278,7 @@ "arg2": "value2", "--conf": "valueConf" }, + "executionClass": "STANDARD", "executionProperty": { "maxConcurrentRuns": 2 }, @@ -296,8 +297,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "SuccessMetricRule": { @@ -345,20 +346,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_events.CfnRule", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_events.Rule", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Job", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "StreamingJob2.0": { @@ -373,8 +374,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -412,8 +413,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "DefaultPolicy": { @@ -481,20 +482,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -540,14 +541,14 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Job", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "EtlJob3.0": { @@ -562,8 +563,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -601,8 +602,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "DefaultPolicy": { @@ -707,20 +708,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "SparkUIBucket": { @@ -735,14 +736,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.CfnBucket", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.Bucket", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -794,6 +795,7 @@ "arg2": "value2", "--conf": "valueConf" }, + "executionClass": "STANDARD", "executionProperty": { "maxConcurrentRuns": 2 }, @@ -812,8 +814,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "SuccessMetricRule": { @@ -861,20 +863,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_events.CfnRule", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_events.Rule", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Job", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "StreamingJob3.0": { @@ -889,8 +891,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -928,8 +930,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "DefaultPolicy": { @@ -997,20 +999,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -1056,14 +1058,14 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Job", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "EtlJob4.0": { @@ -1078,8 +1080,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -1117,8 +1119,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "DefaultPolicy": { @@ -1223,20 +1225,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "SparkUIBucket": { @@ -1251,14 +1253,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.CfnBucket", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.Bucket", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -1310,6 +1312,7 @@ "arg2": "value2", "--conf": "valueConf" }, + "executionClass": "STANDARD", "executionProperty": { "maxConcurrentRuns": 2 }, @@ -1328,8 +1331,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "SuccessMetricRule": { @@ -1377,20 +1380,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_events.CfnRule", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_events.Rule", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Job", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "StreamingJob4.0": { @@ -1405,8 +1408,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -1444,8 +1447,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "DefaultPolicy": { @@ -1513,20 +1516,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -1572,14 +1575,14 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Job", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "ShellJob": { @@ -1594,8 +1597,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -1633,8 +1636,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "DefaultPolicy": { @@ -1702,20 +1705,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -1759,14 +1762,14 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Job", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "ShellJob39": { @@ -1781,8 +1784,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob39/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -1820,8 +1823,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "DefaultPolicy": { @@ -1889,20 +1892,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -1946,14 +1949,14 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Job", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "RayJob": { @@ -1968,8 +1971,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/RayJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -2007,8 +2010,8 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "DefaultPolicy": { @@ -2076,20 +2079,20 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Resource": { @@ -2136,36 +2139,221 @@ } }, "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Job", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" + } + }, + "EtlJobWithFLEX": { + "id": "EtlJobWithFLEX", + "path": "aws-glue-job/EtlJobWithFLEX", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-glue-job/EtlJobWithFLEX/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/EtlJobWithFLEX/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.55" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJobWithFLEX/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.55" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-glue-job/EtlJobWithFLEX/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJobWithFLEX/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "EtlJobWithFLEXServiceRoleDefaultPolicyEA63C5FE", + "roles": [ + { + "Ref": "EtlJobWithFLEXServiceRoleBA7C99A5" + } + ] + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.55" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.55" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.55" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJobWithFLEX/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "glueetl", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, + "defaultArguments": { + "--job-language": "python" + }, + "executionClass": "FLEX", + "glueVersion": "3.0", + "name": "EtlJobWithFLEX", + "numberOfWorkers": 10, + "workerType": "G.1X" + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.55" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "BootstrapVersion": { "id": "BootstrapVersion", "path": "aws-glue-job/BootstrapVersion", "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "CheckBootstrapVersion": { "id": "CheckBootstrapVersion", "path": "aws-glue-job/CheckBootstrapVersion", "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } }, "Tree": { @@ -2173,13 +2361,13 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.26" + "version": "10.2.55" } } }, "constructInfo": { - "fqn": "aws-cdk-lib.App", - "version": "0.0.0" + "fqn": "constructs.Construct", + "version": "10.2.55" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts index 6ae1dd8074dad..aa420c52eccf4 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts @@ -50,6 +50,7 @@ const script = glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world. quiet: true, logStreamPrefix: 'EtlJob', }, + executionClass: glue.ExecutionClass.STANDARD, tags: { key: 'value', }, @@ -125,4 +126,16 @@ new glue.Job(stack, 'RayJob', { }, }); +new glue.Job(stack, 'EtlJobWithFLEX', { + jobName: 'EtlJobWithFLEX', + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V3_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + workerType: glue.WorkerType.G_1X, + workerCount: 10, + executionClass: glue.ExecutionClass.FLEX, +}); + app.synth(); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index 171857e5b5668..59316670b0410 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -306,6 +306,162 @@ describe('Job', () => { }); }); + describe('enabling execution class', () => { + describe('enabling execution class with FLEX', () => { + beforeEach(() => { + job = new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V3_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + executionClass: glue.ExecutionClass.FLEX, + }); + }); + + test('should set FLEX', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + ExecutionClass: 'FLEX', + }); + }); + }); + + describe('enabling execution class with FLEX and WorkerType G_1X', () => { + beforeEach(() => { + job = new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V3_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + executionClass: glue.ExecutionClass.FLEX, + workerType: glue.WorkerType.G_1X, + workerCount: 10, + }); + }); + + test('should set FLEX', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + ExecutionClass: 'FLEX', + WorkerType: 'G.1X', + }); + }); + }); + + describe('enabling execution class with FLEX and WorkerType G_2X', () => { + beforeEach(() => { + job = new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V3_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + executionClass: glue.ExecutionClass.FLEX, + workerType: glue.WorkerType.G_2X, + workerCount: 10, + }); + }); + + test('should set FLEX', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + ExecutionClass: 'FLEX', + WorkerType: 'G.2X', + }); + }); + }); + + describe('enabling execution class with STANDARD', () => { + beforeEach(() => { + job = new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V3_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + executionClass: glue.ExecutionClass.STANDARD, + }); + }); + + test('should set STANDARD', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + ExecutionClass: 'STANDARD', + }); + }); + }); + + describe('errors for execution class with FLEX', () => { + test('job type except JobType.ETL should throw', () => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonShell({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + executionClass: glue.ExecutionClass.FLEX, + })).toThrow('FLEX ExecutionClass is only available for JobType.ETL jobs'); + }); + + test('with glue version 0.9 should throw', () => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V0_9, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + executionClass: glue.ExecutionClass.FLEX, + })).toThrow('FLEX ExecutionClass is only available for GlueVersion 3.0 or later'); + }); + + test('with glue version 1.0 should throw', () => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V1_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + executionClass: glue.ExecutionClass.FLEX, + })).toThrow('FLEX ExecutionClass is only available for GlueVersion 3.0 or later'); + }); + + test('with glue version 2.0 should throw', () => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + executionClass: glue.ExecutionClass.FLEX, + })).toThrow('FLEX ExecutionClass is only available for GlueVersion 3.0 or later'); + }); + + test('with G_025X as worker type that is neither G_1X nor G_2X should throw', () => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V3_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + workerType: glue.WorkerType.G_025X, + workerCount: 2, + executionClass: glue.ExecutionClass.FLEX, + })).toThrow('FLEX ExecutionClass is only available for WorkerType G_1X or G_2X'); + }); + + test('with G_4X as worker type that is neither G_1X nor G_2X should throw', () => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V3_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + workerType: glue.WorkerType.G_4X, + workerCount: 10, + executionClass: glue.ExecutionClass.FLEX, + })).toThrow('FLEX ExecutionClass is only available for WorkerType G_1X or G_2X'); + }); + }); + }); + describe('enabling spark ui', () => { describe('with no bucket or path provided', () => { beforeEach(() => {