Merge branch 'main' into redshift_elastic_resizing

aws · Jul 12, 2022 · 02b7948 · 02b7948
2 parents 727b824 + f71144f
commit 02b7948
Show file tree

Hide file tree

Showing 17 changed files with 364 additions and 67 deletions.
diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md
@@ -264,6 +264,31 @@ myTable.addPartitionIndex({
 });
 ```
 
+### Partition Filtering
+
+If you have a table with a large number of partitions that grows over time, consider using AWS Glue partition indexing and filtering.
+
+```ts
+declare const myDatabase: glue.Database;
+new glue.Table(this, 'MyTable', {
+    database: myDatabase,
+    tableName: 'my_table',
+    columns: [{
+        name: 'col1',
+        type: glue.Schema.STRING,
+    }],
+    partitionKeys: [{
+        name: 'year',
+        type: glue.Schema.SMALL_INT,
+    }, {
+        name: 'month',
+        type: glue.Schema.SMALL_INT,
+    }],
+    dataFormat: glue.DataFormat.JSON,
+    enablePartitionFiltering: true,
+});
+```
+
 ## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html)
 
 You can enable encryption on a Table's data:

diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts
@@ -172,6 +172,15 @@ export interface TableProps {
    * @default false
    */
   readonly storedAsSubDirectories?: boolean;
+
+  /**
+   * Enables partition filtering.
+   *
+   * @see https://docs.aws.amazon.com/athena/latest/ug/glue-best-practices.html#glue-best-practices-partition-index
+   *
+   * @default - The parameter is not defined
+   */
+  readonly enablePartitionFiltering?: boolean;
 }
 
 /**
@@ -302,8 +311,9 @@ export class Table extends Resource implements ITable {
         partitionKeys: renderColumns(props.partitionKeys),
 
         parameters: {
-          classification: props.dataFormat.classificationString?.value,
-          has_encrypted_data: this.encryption !== TableEncryption.UNENCRYPTED,
+          'classification': props.dataFormat.classificationString?.value,
+          'has_encrypted_data': this.encryption !== TableEncryption.UNENCRYPTED,
+          'partition_filtering.enabled': props.enablePartitionFiltering,
         },
         storageDescriptor: {
           location: `s3://${this.bucket.bucketName}/${this.s3Prefix}`,

diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts
@@ -87,6 +87,14 @@ const encryptedTable = new glue.Table(stack, 'MyEncryptedTable', {
   encryptionKey: new kms.Key(stack, 'MyKey'),
 });
 
+new glue.Table(stack, 'MyPartitionFilteredTable', {
+  database,
+  tableName: 'partition_filtered_table',
+  columns,
+  dataFormat: glue.DataFormat.JSON,
+  enablePartitionFiltering: true,
+});
+
 const user = new iam.User(stack, 'MyUser');
 csvTable.grantReadWrite(user);
 encryptedTable.grantReadWrite(user);

diff --git a/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/aws-cdk-glue.assets.json
@@ -1,15 +1,15 @@
 {
-  "version": "17.0.0",
+  "version": "20.0.0",
   "files": {
-    "92638b7a8efe38efd7c845883423f3767018a9e5bd3d67d8d638332f054d0d0f": {
+    "419b39f03d496de4fb02e795181e9a2ab218fb90bf7a5c9354cf93baa6fea2cf": {
       "source": {
         "path": "aws-cdk-glue.template.json",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "92638b7a8efe38efd7c845883423f3767018a9e5bd3d67d8d638332f054d0d0f.json",
+          "objectKey": "419b39f03d496de4fb02e795181e9a2ab218fb90bf7a5c9354cf93baa6fea2cf.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }

diff --git a/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/aws-cdk-glue.template.json
@@ -423,6 +423,76 @@
     }
    }
   },
+  "MyPartitionFilteredTableBucket6ACAA137": {
+   "Type": "AWS::S3::Bucket",
+   "UpdateReplacePolicy": "Retain",
+   "DeletionPolicy": "Retain"
+  },
+  "MyPartitionFilteredTable324BA27A": {
+   "Type": "AWS::Glue::Table",
+   "Properties": {
+    "CatalogId": {
+     "Ref": "AWS::AccountId"
+    },
+    "DatabaseName": {
+     "Ref": "MyDatabase1E2517DB"
+    },
+    "TableInput": {
+     "Description": "partition_filtered_table generated by CDK",
+     "Name": "partition_filtered_table",
+     "Parameters": {
+      "classification": "json",
+      "has_encrypted_data": false,
+      "partition_filtering.enabled": true
+     },
+     "StorageDescriptor": {
+      "Columns": [
+       {
+        "Name": "col1",
+        "Type": "string"
+       },
+       {
+        "Comment": "col2 comment",
+        "Name": "col2",
+        "Type": "string"
+       },
+       {
+        "Name": "col3",
+        "Type": "array<string>"
+       },
+       {
+        "Name": "col4",
+        "Type": "map<string,string>"
+       },
+       {
+        "Name": "col5",
+        "Type": "struct<col1:string>"
+       }
+      ],
+      "Compressed": false,
+      "InputFormat": "org.apache.hadoop.mapred.TextInputFormat",
+      "Location": {
+       "Fn::Join": [
+        "",
+        [
+         "s3://",
+         {
+          "Ref": "MyPartitionFilteredTableBucket6ACAA137"
+         },
+         "/"
+        ]
+       ]
+      },
+      "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+      "SerdeInfo": {
+       "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe"
+      },
+      "StoredAsSubDirectories": false
+     },
+     "TableType": "EXTERNAL_TABLE"
+    }
+   }
+  },
   "MyUserDC45028B": {
    "Type": "AWS::IAM::User"
   },

diff --git a/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/cdk.out b/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/cdk.out
@@ -1 +1 @@
-{"version":"17.0.0"}
+{"version":"20.0.0"}
diff --git a/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/integ.json b/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/integ.json
@@ -1,7 +1,7 @@
 {
-  "version": "18.0.0",
+  "version": "20.0.0",
   "testCases": {
-    "aws-glue/test/integ.table": {
+    "integ.table": {
       "stacks": [
         "aws-cdk-glue"
       ],

diff --git a/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/manifest.json
@@ -1,5 +1,5 @@
 {
-  "version": "17.0.0",
+  "version": "20.0.0",
   "artifacts": {
     "Tree": {
       "type": "cdk:tree",
@@ -69,6 +69,18 @@
             "data": "MyEncryptedTable981A88C6"
           }
         ],
+        "/aws-cdk-glue/MyPartitionFilteredTable/Bucket/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "MyPartitionFilteredTableBucket6ACAA137"
+          }
+        ],
+        "/aws-cdk-glue/MyPartitionFilteredTable/Table": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "MyPartitionFilteredTable324BA27A"
+          }
+        ],
         "/aws-cdk-glue/MyUser/Resource": [
           {
             "type": "aws:cdk:logicalId",

diff --git a/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/tree.json b/packages/@aws-cdk/aws-glue/test/table.integ.snapshot/tree.json
@@ -9,7 +9,7 @@
         "path": "Tree",
         "constructInfo": {
           "fqn": "constructs.Construct",
-          "version": "10.0.9"
+          "version": "10.1.33"
         }
       },
       "aws-cdk-glue": {
@@ -596,6 +596,111 @@
               "version": "0.0.0"
             }
           },
+          "MyPartitionFilteredTable": {
+            "id": "MyPartitionFilteredTable",
+            "path": "aws-cdk-glue/MyPartitionFilteredTable",
+            "children": {
+              "Bucket": {
+                "id": "Bucket",
+                "path": "aws-cdk-glue/MyPartitionFilteredTable/Bucket",
+                "children": {
+                  "Resource": {
+                    "id": "Resource",
+                    "path": "aws-cdk-glue/MyPartitionFilteredTable/Bucket/Resource",
+                    "attributes": {
+                      "aws:cdk:cloudformation:type": "AWS::S3::Bucket",
+                      "aws:cdk:cloudformation:props": {}
+                    },
+                    "constructInfo": {
+                      "fqn": "@aws-cdk/aws-s3.CfnBucket",
+                      "version": "0.0.0"
+                    }
+                  }
+                },
+                "constructInfo": {
+                  "fqn": "@aws-cdk/aws-s3.Bucket",
+                  "version": "0.0.0"
+                }
+              },
+              "Table": {
+                "id": "Table",
+                "path": "aws-cdk-glue/MyPartitionFilteredTable/Table",
+                "attributes": {
+                  "aws:cdk:cloudformation:type": "AWS::Glue::Table",
+                  "aws:cdk:cloudformation:props": {
+                    "catalogId": {
+                      "Ref": "AWS::AccountId"
+                    },
+                    "databaseName": {
+                      "Ref": "MyDatabase1E2517DB"
+                    },
+                    "tableInput": {
+                      "name": "partition_filtered_table",
+                      "description": "partition_filtered_table generated by CDK",
+                      "parameters": {
+                        "classification": "json",
+                        "has_encrypted_data": false,
+                        "partition_filtering.enabled": true
+                      },
+                      "storageDescriptor": {
+                        "location": {
+                          "Fn::Join": [
+                            "",
+                            [
+                              "s3://",
+                              {
+                                "Ref": "MyPartitionFilteredTableBucket6ACAA137"
+                              },
+                              "/"
+                            ]
+                          ]
+                        },
+                        "compressed": false,
+                        "storedAsSubDirectories": false,
+                        "columns": [
+                          {
+                            "name": "col1",
+                            "type": "string"
+                          },
+                          {
+                            "name": "col2",
+                            "type": "string",
+                            "comment": "col2 comment"
+                          },
+                          {
+                            "name": "col3",
+                            "type": "array<string>"
+                          },
+                          {
+                            "name": "col4",
+                            "type": "map<string,string>"
+                          },
+                          {
+                            "name": "col5",
+                            "type": "struct<col1:string>"
+                          }
+                        ],
+                        "inputFormat": "org.apache.hadoop.mapred.TextInputFormat",
+                        "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+                        "serdeInfo": {
+                          "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe"
+                        }
+                      },
+                      "tableType": "EXTERNAL_TABLE"
+                    }
+                  }
+                },
+                "constructInfo": {
+                  "fqn": "@aws-cdk/aws-glue.CfnTable",
+                  "version": "0.0.0"
+                }
+              }
+            },
+            "constructInfo": {
+              "fqn": "@aws-cdk/aws-glue.Table",
+              "version": "0.0.0"
+            }
+          },
           "MyUser": {
             "id": "MyUser",
             "path": "aws-cdk-glue/MyUser",