Skip to content

Commit

Permalink
feat(glue): support glue version 4.0 (aws#23223)
Browse files Browse the repository at this point in the history
https://aws.amazon.com/blogs/aws/new-aws-glue-4-0-new-and-updated-engines-more-data-formats-and-more/

closes aws#23220

----

### All Submissions:

* [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Construct Runtime Dependencies:

* [ ] This PR adds new construct runtime dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-construct-runtime-dependencies)

### New Features

* [x] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [x] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
  • Loading branch information
humanzz authored and Brennan Ho committed Feb 22, 2023
1 parent b0f9827 commit 1ba3f90
Show file tree
Hide file tree
Showing 10 changed files with 2,127 additions and 168 deletions.
4 changes: 2 additions & 2 deletions packages/@aws-cdk/aws-glue/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ An ETL job processes data in batches using Apache Spark.
declare const bucket: s3.Bucket;
new glue.Job(this, 'ScalaSparkEtlJob', {
executable: glue.JobExecutable.scalaEtl({
glueVersion: glue.GlueVersion.V2_0,
glueVersion: glue.GlueVersion.V4_0,
script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'),
className: 'com.example.HelloWorld',
extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')],
Expand All @@ -61,7 +61,7 @@ A Streaming job is similar to an ETL job, except that it performs ETL on data st
```ts
new glue.Job(this, 'PythonSparkStreamingJob', {
executable: glue.JobExecutable.pythonStreaming({
glueVersion: glue.GlueVersion.V2_0,
glueVersion: glue.GlueVersion.V4_0,
pythonVersion: glue.PythonVersion.THREE,
script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),
}),
Expand Down
7 changes: 6 additions & 1 deletion packages/@aws-cdk/aws-glue/lib/job-executable.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ export class GlueVersion {
*/
public static readonly V3_0 = new GlueVersion('3.0');

/**
* Glue version using Spark 3.3.0 and Python 3.10
*/
public static readonly V4_0 = new GlueVersion('4.0');

/**
* Custom Glue version
* @param version custom version
Expand Down Expand Up @@ -292,7 +297,7 @@ export class JobExecutable {
if (config.language !== JobLanguage.PYTHON) {
throw new Error('Python shell requires the language to be set to Python');
}
if ([GlueVersion.V0_9, GlueVersion.V2_0, GlueVersion.V3_0].includes(config.glueVersion)) {
if ([GlueVersion.V0_9, GlueVersion.V2_0, GlueVersion.V3_0, GlueVersion.V4_0].includes(config.glueVersion)) {
throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support Python Shell`);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"version": "21.0.0",
"version": "22.0.0",
"files": {
"432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": {
"source": {
Expand All @@ -14,15 +14,15 @@
}
}
},
"3d4fb124b6f262a7f5f621715b1437dcf5af3450121d622990f89e94fd55d8f5": {
"0985af21379e9d6e1cba091105ecb533ee38a96b4c738816daf17d951a0752b4": {
"source": {
"path": "aws-glue-job.template.json",
"packaging": "file"
},
"destinations": {
"current_account-current_region": {
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
"objectKey": "3d4fb124b6f262a7f5f621715b1437dcf5af3450121d622990f89e94fd55d8f5.json",
"objectKey": "0985af21379e9d6e1cba091105ecb533ee38a96b4c738816daf17d951a0752b4.json",
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
}
}
Expand Down
Loading

0 comments on commit 1ba3f90

Please sign in to comment.