From d17a108cc431cb8c3d233bcae4c60998c6c599b6 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Sun, 10 Jan 2021 12:30:27 +0000 Subject: [PATCH 01/12] feat(glue): add Connection construct (#12442) closes #12442 --- packages/@aws-cdk/aws-glue/README.md | 14 ++ packages/@aws-cdk/aws-glue/lib/connection.ts | 170 ++++++++++++++++++ packages/@aws-cdk/aws-glue/lib/index.ts | 1 + packages/@aws-cdk/aws-glue/package.json | 2 + .../@aws-cdk/aws-glue/test/connection.test.ts | 112 ++++++++++++ 5 files changed, 299 insertions(+) create mode 100644 packages/@aws-cdk/aws-glue/lib/connection.ts create mode 100644 packages/@aws-cdk/aws-glue/test/connection.test.ts diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index 77ee00834ddd7..9c0f47b2ea018 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -23,6 +23,20 @@ This module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) project. +## Connection + +A `Connection` allows Glue jobs, crawlers and development endpoints to access certain types of data stores. + +```ts +new glue.Connection(stack, 'MyConnection', { + connectionType: glue.ConnectionTypes.NETWORK, + securityGroups: [securityGroup], + subnet, +}); +``` + +See [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations. + ## Database A `Database` is a logical grouping of `Tables` in the Glue Catalog. diff --git a/packages/@aws-cdk/aws-glue/lib/connection.ts b/packages/@aws-cdk/aws-glue/lib/connection.ts new file mode 100644 index 0000000000000..407e9eef9d0a3 --- /dev/null +++ b/packages/@aws-cdk/aws-glue/lib/connection.ts @@ -0,0 +1,170 @@ +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as cdk from '@aws-cdk/core'; +import * as constructs from 'constructs'; +import { CfnConnection } from './glue.generated'; + +/** + * The type of the glue connection + */ +export enum ConnectionType { + /** + * Designates a connection to a database through Java Database Connectivity (JDBC). + */ + JDBC = 'JDBC', + + /** + * Designates a connection to an Apache Kafka streaming platform. + */ + KAFKA = 'KAFKA', + + /** + * Designates a connection to a MongoDB document database. + */ + MONGODB = 'MONGODB', + + /** + * Designates a network connection to a data source within an Amazon Virtual Private Cloud environment (Amazon VPC). + */ + NETWORK = 'NETWORK' +} + +/** + * Interface representing a created or an imported {@link Connection} + */ +export interface IConnection extends cdk.IResource { + /** + * The name of the connection + * @attribute + */ + readonly connectionName: string; + + /** + * The ARN of the connection + * @attribute + */ + readonly connectionArn: string; +} + +/** + * Attributes for importing {@link Connection} + */ +export interface ConnectionAttributes { + /** + * The name of the connection + */ + readonly connectionName: string; +} + +/** + * Construction properties for {@link Connection} + */ +export interface ConnectionProps { + /** + * The name of the connection + * @default cloudformation generated name + */ + readonly connectionName?: string; + + /** + * The description of the connection. + * @default no description + */ + readonly description?: string; + + /** + * The type of the connection + */ + readonly connectionType: ConnectionType; + + /** + * Key-Value pairs that define parameters for the connection. + * @default empty properties + */ + readonly connectionProperties?: { [key: string]: string }; + + /** + * A list of criteria that can be used in selecting this connection. + * @default no match criteria + */ + readonly matchCriteria?: string[]; + + /** + * The list of security groups needed to successfully make this connection e.g. to successfully connect to VPC. + * @default no security group + */ + readonly securityGroups?: ec2.ISecurityGroup[]; + + /** + * The VPC subnet to connect to resources within a VPC. See more at https://docs.aws.amazon.com/glue/latest/dg/start-connecting.html. + * @default no subnet + */ + readonly subnet?: ec2.ISubnet; +} + +/** + * An AWS Glue connection to a data source. + */ +export class Connection extends cdk.Resource implements IConnection { + + /** + * Creates a Connection construct that represents an external connection. + * + * @param scope The scope creating construct (usually `this`). + * @param id The construct's id. + * @param attrs Import attributes + */ + public static fromConnectionAttributes(scope: constructs.Construct, id: string, attrs: ConnectionAttributes): IConnection { + class Import extends cdk.Resource implements IConnection { + public readonly connectionName = attrs.connectionName; + public readonly connectionArn = Connection.buildConnectionArn(scope, attrs.connectionName); + } + + return new Import(scope, id); + } + + private static buildConnectionArn(scope: constructs.Construct, connectionName: string) : string { + return cdk.Stack.of(scope).formatArn({ + service: 'glue', + resource: 'connection', + resourceName: connectionName, + }); + } + + /** + * The ARN of the connection + */ + public readonly connectionArn: string; + + /** + * The name of the connection + */ + public readonly connectionName: string; + + constructor(scope: constructs.Construct, id: string, props: ConnectionProps) { + super(scope, id, { + physicalName: props.connectionName, + }); + + const physicalConnectionRequirements = props.subnet || props.securityGroups ? { + availabilityZone: props.subnet ? props.subnet.availabilityZone : undefined, + subnetId: props.subnet ? props.subnet.subnetId : undefined, + securityGroupIdList: props.securityGroups ? props.securityGroups.map(sg => sg.securityGroupId) : undefined, + } : undefined; + + const connectionResource = new CfnConnection(this, 'Resource', { + catalogId: cdk.Aws.ACCOUNT_ID, + connectionInput: { + connectionProperties: props.connectionProperties, + connectionType: props.connectionType, + description: props.description, + matchCriteria: props.matchCriteria, + name: props.connectionName, + physicalConnectionRequirements, + }, + }); + + const resourceName = this.getResourceNameAttribute(connectionResource.ref); + this.connectionArn = Connection.buildConnectionArn(this, resourceName); + this.connectionName = resourceName; + } +} diff --git a/packages/@aws-cdk/aws-glue/lib/index.ts b/packages/@aws-cdk/aws-glue/lib/index.ts index f0370c21041b2..4880b3b7136b8 100644 --- a/packages/@aws-cdk/aws-glue/lib/index.ts +++ b/packages/@aws-cdk/aws-glue/lib/index.ts @@ -1,6 +1,7 @@ // AWS::Glue CloudFormation Resources: export * from './glue.generated'; +export * from './connection'; export * from './database'; export * from './schema'; export * from './data-format'; diff --git a/packages/@aws-cdk/aws-glue/package.json b/packages/@aws-cdk/aws-glue/package.json index ca6ed1bb731a5..22243f7922478 100644 --- a/packages/@aws-cdk/aws-glue/package.json +++ b/packages/@aws-cdk/aws-glue/package.json @@ -80,6 +80,7 @@ "pkglint": "0.0.0" }, "dependencies": { + "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", @@ -88,6 +89,7 @@ }, "homepage": "https://github.com/aws/aws-cdk", "peerDependencies": { + "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", diff --git a/packages/@aws-cdk/aws-glue/test/connection.test.ts b/packages/@aws-cdk/aws-glue/test/connection.test.ts new file mode 100644 index 0000000000000..6331bce8d551c --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/connection.test.ts @@ -0,0 +1,112 @@ +import { strictEqual } from 'assert'; +import { expect, haveResource } from '@aws-cdk/assert'; +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as cdk from '@aws-cdk/core'; +import '@aws-cdk/assert/jest'; +import * as glue from '../lib'; + +test('a connection with connection properties', () => { + const stack = new cdk.Stack(); + new glue.Connection(stack, 'Connection', { + connectionType: glue.ConnectionType.JDBC, + connectionProperties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + expect(stack).to(haveResource('AWS::Glue::Connection', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + ConnectionInput: { + ConnectionProperties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + ConnectionType: 'JDBC', + }, + })); +}); + +test('a connection with a subnet and security group', () => { + const stack = new cdk.Stack(); + const subnet = ec2.Subnet.fromSubnetAttributes(stack, 'subnet', { + subnetId: 'subnetId', + availabilityZone: 'azId', + }); + const securityGroup = ec2.SecurityGroup.fromSecurityGroupId(stack, 'securityGroup', 'sgId'); + new glue.Connection(stack, 'Connection', { + connectionType: glue.ConnectionType.NETWORK, + securityGroups: [securityGroup], + subnet, + }); + + expect(stack).to(haveResource('AWS::Glue::Connection', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + ConnectionInput: { + ConnectionType: 'NETWORK', + PhysicalConnectionRequirements: { + AvailabilityZone: 'azId', + SubnetId: 'subnetId', + SecurityGroupIdList: ['sgId'], + }, + }, + })); +}); + +test('a connection with a name and description', () => { + const stack = new cdk.Stack(); + new glue.Connection(stack, 'Connection', { + connectionName: 'name', + description: 'description', + connectionType: glue.ConnectionType.NETWORK, + }); + + expect(stack).to(haveResource('AWS::Glue::Connection', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + ConnectionInput: { + ConnectionType: 'NETWORK', + Name: 'name', + Description: 'description', + }, + })); +}); + +test('a connection with match criteria', () => { + const stack = new cdk.Stack(); + new glue.Connection(stack, 'Connection', { + connectionType: glue.ConnectionType.NETWORK, + matchCriteria: ['c1', 'c2'], + }); + + expect(stack).to(haveResource('AWS::Glue::Connection', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + ConnectionInput: { + ConnectionType: 'NETWORK', + MatchCriteria: ['c1', 'c2'], + }, + })); +}); + +test('fromConnectionAttributes', () => { + const stack = new cdk.Stack(); + const connection = glue.Connection.fromConnectionAttributes(stack, 'ImportedConnection', { + connectionName: 'name', + }); + + strictEqual(connection.connectionName, 'name'); + strictEqual(connection.connectionArn, stack.formatArn({ + service: 'glue', + resource: 'connection', + resourceName: 'name', + })); +}); \ No newline at end of file From e1d2db96af5cc150afd4b7cd1435eaaeb7c51046 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Sun, 10 Jan 2021 12:47:31 +0000 Subject: [PATCH 02/12] future-proof ConnectionType enumeration follow the pattern in Lambda's Runtime to use a class with static members instead of enum --- packages/@aws-cdk/aws-glue/lib/connection.ts | 35 ++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/lib/connection.ts b/packages/@aws-cdk/aws-glue/lib/connection.ts index 407e9eef9d0a3..2f4913135350c 100644 --- a/packages/@aws-cdk/aws-glue/lib/connection.ts +++ b/packages/@aws-cdk/aws-glue/lib/connection.ts @@ -5,27 +5,50 @@ import { CfnConnection } from './glue.generated'; /** * The type of the glue connection + * + * If you need to use a connection type that doesn't exist as a static member, you + * can instantiate a `ConnectionType` object, e.g: `new ConnectionType('NEW_TYPE')`. */ -export enum ConnectionType { +export class ConnectionType { + /** A list of all known `ConnectionType`s. */ + public static readonly ALL = new Array(); + /** * Designates a connection to a database through Java Database Connectivity (JDBC). */ - JDBC = 'JDBC', + public static readonly JDBC = new ConnectionType('JDBC'); /** * Designates a connection to an Apache Kafka streaming platform. */ - KAFKA = 'KAFKA', + public static readonly KAFKA = new ConnectionType('KAFKA'); /** * Designates a connection to a MongoDB document database. */ - MONGODB = 'MONGODB', + public static readonly MONGODB = new ConnectionType('MONGODB'); /** * Designates a network connection to a data source within an Amazon Virtual Private Cloud environment (Amazon VPC). */ - NETWORK = 'NETWORK' + public static readonly NETWORK = new ConnectionType('NETWORK'); + + /** + * The name of this ConnectionType, as expected by Connection resource. + */ + public readonly name: string; + + constructor(name: string) { + this.name = name; + ConnectionType.ALL.push(this); + } + + /** + * The connection type name as expected by Connection resource. + */ + public toString(): string { + return this.name; + } } /** @@ -155,7 +178,7 @@ export class Connection extends cdk.Resource implements IConnection { catalogId: cdk.Aws.ACCOUNT_ID, connectionInput: { connectionProperties: props.connectionProperties, - connectionType: props.connectionType, + connectionType: props.connectionType.name, description: props.description, matchCriteria: props.matchCriteria, name: props.connectionName, From 275f2f122a4f98e0e73f9cd8d1163386a3cbd6d5 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Sun, 10 Jan 2021 14:57:30 +0000 Subject: [PATCH 03/12] test connection with custom type --- .../@aws-cdk/aws-glue/test/connection.test.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/@aws-cdk/aws-glue/test/connection.test.ts b/packages/@aws-cdk/aws-glue/test/connection.test.ts index 6331bce8d551c..03f63261a32b7 100644 --- a/packages/@aws-cdk/aws-glue/test/connection.test.ts +++ b/packages/@aws-cdk/aws-glue/test/connection.test.ts @@ -79,6 +79,26 @@ test('a connection with a name and description', () => { })); }); +test('a connection with a custom type', () => { + const stack = new cdk.Stack(); + new glue.Connection(stack, 'Connection', { + connectionName: 'name', + description: 'description', + connectionType: new glue.ConnectionType('CUSTOM_TYPE'), + }); + + expect(stack).to(haveResource('AWS::Glue::Connection', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + ConnectionInput: { + ConnectionType: 'CUSTOM_TYPE', + Name: 'name', + Description: 'description', + }, + })); +}); + test('a connection with match criteria', () => { const stack = new cdk.Stack(); new glue.Connection(stack, 'Connection', { From 5651632ae3d720b1f3dedf9d56b9a4cd5ff04c3f Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Mon, 11 Jan 2021 15:33:56 +0000 Subject: [PATCH 04/12] add network connection integration test --- .../test/integ.connection.expected.json | 559 ++++++++++++++++++ .../aws-glue/test/integ.connection.ts | 20 + 2 files changed, 579 insertions(+) create mode 100644 packages/@aws-cdk/aws-glue/test/integ.connection.expected.json create mode 100644 packages/@aws-cdk/aws-glue/test/integ.connection.ts diff --git a/packages/@aws-cdk/aws-glue/test/integ.connection.expected.json b/packages/@aws-cdk/aws-glue/test/integ.connection.expected.json new file mode 100644 index 0000000000000..3422ce4ff408b --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/integ.connection.expected.json @@ -0,0 +1,559 @@ +{ + "Resources": { + "Vpc8378EB38": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "InstanceTenancy": "default", + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc" + } + ] + } + }, + "VpcPublicSubnet1Subnet5C2D37C4": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.0.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet1" + } + ] + } + }, + "VpcPublicSubnet1RouteTable6C95E38E": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet1" + } + ] + } + }, + "VpcPublicSubnet1RouteTableAssociation97140677": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet1RouteTable6C95E38E" + }, + "SubnetId": { + "Ref": "VpcPublicSubnet1Subnet5C2D37C4" + } + } + }, + "VpcPublicSubnet1DefaultRoute3DA9E72A": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet1RouteTable6C95E38E" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "VpcIGWD7BA715C" + } + }, + "DependsOn": [ + "VpcVPCGWBF912B6E" + ] + }, + "VpcPublicSubnet1EIPD7E02669": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet1" + } + ] + } + }, + "VpcPublicSubnet1NATGateway4D7517AA": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "VpcPublicSubnet1EIPD7E02669", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "VpcPublicSubnet1Subnet5C2D37C4" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet1" + } + ] + } + }, + "VpcPublicSubnet2Subnet691E08A3": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.32.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet2" + } + ] + } + }, + "VpcPublicSubnet2RouteTable94F7E489": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet2" + } + ] + } + }, + "VpcPublicSubnet2RouteTableAssociationDD5762D8": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet2RouteTable94F7E489" + }, + "SubnetId": { + "Ref": "VpcPublicSubnet2Subnet691E08A3" + } + } + }, + "VpcPublicSubnet2DefaultRoute97F91067": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet2RouteTable94F7E489" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "VpcIGWD7BA715C" + } + }, + "DependsOn": [ + "VpcVPCGWBF912B6E" + ] + }, + "VpcPublicSubnet2EIP3C605A87": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet2" + } + ] + } + }, + "VpcPublicSubnet2NATGateway9182C01D": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "VpcPublicSubnet2EIP3C605A87", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "VpcPublicSubnet2Subnet691E08A3" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet2" + } + ] + } + }, + "VpcPublicSubnet3SubnetBE12F0B6": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.64.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet3" + } + ] + } + }, + "VpcPublicSubnet3RouteTable93458DBB": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet3" + } + ] + } + }, + "VpcPublicSubnet3RouteTableAssociation1F1EDF02": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet3RouteTable93458DBB" + }, + "SubnetId": { + "Ref": "VpcPublicSubnet3SubnetBE12F0B6" + } + } + }, + "VpcPublicSubnet3DefaultRoute4697774F": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet3RouteTable93458DBB" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "VpcIGWD7BA715C" + } + }, + "DependsOn": [ + "VpcVPCGWBF912B6E" + ] + }, + "VpcPublicSubnet3EIP3A666A23": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet3" + } + ] + } + }, + "VpcPublicSubnet3NATGateway7640CD1D": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "VpcPublicSubnet3EIP3A666A23", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "VpcPublicSubnet3SubnetBE12F0B6" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PublicSubnet3" + } + ] + } + }, + "VpcPrivateSubnet1Subnet536B997A": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.96.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PrivateSubnet1" + } + ] + } + }, + "VpcPrivateSubnet1RouteTableB2C5B500": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PrivateSubnet1" + } + ] + } + }, + "VpcPrivateSubnet1RouteTableAssociation70C59FA6": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet1RouteTableB2C5B500" + }, + "SubnetId": { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + } + } + }, + "VpcPrivateSubnet1DefaultRouteBE02A9ED": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet1RouteTableB2C5B500" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "VpcPublicSubnet1NATGateway4D7517AA" + } + } + }, + "VpcPrivateSubnet2Subnet3788AAA1": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.128.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PrivateSubnet2" + } + ] + } + }, + "VpcPrivateSubnet2RouteTableA678073B": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PrivateSubnet2" + } + ] + } + }, + "VpcPrivateSubnet2RouteTableAssociationA89CAD56": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet2RouteTableA678073B" + }, + "SubnetId": { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + } + } + }, + "VpcPrivateSubnet2DefaultRoute060D2087": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet2RouteTableA678073B" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "VpcPublicSubnet2NATGateway9182C01D" + } + } + }, + "VpcPrivateSubnet3SubnetF258B56E": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.160.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PrivateSubnet3" + } + ] + } + }, + "VpcPrivateSubnet3RouteTableD98824C7": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc/PrivateSubnet3" + } + ] + } + }, + "VpcPrivateSubnet3RouteTableAssociation16BDDC43": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet3RouteTableD98824C7" + }, + "SubnetId": { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + } + }, + "VpcPrivateSubnet3DefaultRoute94B74F0D": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet3RouteTableD98824C7" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "VpcPublicSubnet3NATGateway7640CD1D" + } + } + }, + "VpcIGWD7BA715C": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "aws-glue-connection/Vpc" + } + ] + } + }, + "VpcVPCGWBF912B6E": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "InternetGatewayId": { + "Ref": "VpcIGWD7BA715C" + } + } + }, + "SecurityGroupDD263621": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "aws-glue-connection/SecurityGroup", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "NetworkConnection2B07B7E5": { + "Type": "AWS::Glue::Connection", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "ConnectionInput": { + "ConnectionType": "NETWORK", + "PhysicalConnectionRequirements": { + "AvailabilityZone": "test-region-1a", + "SecurityGroupIdList": [ + { + "Fn::GetAtt": [ + "SecurityGroupDD263621", + "GroupId" + ] + } + ], + "SubnetId": { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.connection.ts b/packages/@aws-cdk/aws-glue/test/integ.connection.ts new file mode 100644 index 0000000000000..4174c660cc568 --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/integ.connection.ts @@ -0,0 +1,20 @@ +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as cdk from '@aws-cdk/core'; +import * as glue from '../lib'; + +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-glue-connection'); + +const vpc = new ec2.Vpc(stack, 'Vpc'); + +const sg = new ec2.SecurityGroup(stack, 'SecurityGroup', { + vpc, +}); + +// Network connection +new glue.Connection(stack, 'NetworkConnection', { + connectionType: glue.ConnectionType.NETWORK, + subnet: vpc.privateSubnets[0], + securityGroups: [sg], +}); From 64916b51449e510a0853150824761882454990b6 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Mon, 15 Feb 2021 09:59:27 +0000 Subject: [PATCH 05/12] address comments --- packages/@aws-cdk/aws-glue/lib/connection.ts | 60 ++++++++++++------- .../@aws-cdk/aws-glue/test/connection.test.ts | 32 ++++++---- .../aws-glue/test/integ.connection.ts | 2 +- 3 files changed, 60 insertions(+), 34 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/lib/connection.ts b/packages/@aws-cdk/aws-glue/lib/connection.ts index 2f4913135350c..9e67c0fbeedc9 100644 --- a/packages/@aws-cdk/aws-glue/lib/connection.ts +++ b/packages/@aws-cdk/aws-glue/lib/connection.ts @@ -10,8 +10,6 @@ import { CfnConnection } from './glue.generated'; * can instantiate a `ConnectionType` object, e.g: `new ConnectionType('NEW_TYPE')`. */ export class ConnectionType { - /** A list of all known `ConnectionType`s. */ - public static readonly ALL = new Array(); /** * Designates a connection to a database through Java Database Connectivity (JDBC). @@ -40,7 +38,6 @@ export class ConnectionType { constructor(name: string) { this.name = name; - ConnectionType.ALL.push(this); } /** @@ -68,16 +65,6 @@ export interface IConnection extends cdk.IResource { readonly connectionArn: string; } -/** - * Attributes for importing {@link Connection} - */ -export interface ConnectionAttributes { - /** - * The name of the connection - */ - readonly connectionName: string; -} - /** * Construction properties for {@link Connection} */ @@ -97,16 +84,17 @@ export interface ConnectionProps { /** * The type of the connection */ - readonly connectionType: ConnectionType; + readonly type: ConnectionType; /** * Key-Value pairs that define parameters for the connection. * @default empty properties */ - readonly connectionProperties?: { [key: string]: string }; + readonly properties?: { [key: string]: string }; /** * A list of criteria that can be used in selecting this connection. + * This is useful for filtering the results of https://awscli.amazonaws.com/v2/documentation/api/latest/reference/glue/get-connections.html * @default no match criteria */ readonly matchCriteria?: string[]; @@ -134,17 +122,47 @@ export class Connection extends cdk.Resource implements IConnection { * * @param scope The scope creating construct (usually `this`). * @param id The construct's id. - * @param attrs Import attributes + * @param connectionArn arn of external connection. */ - public static fromConnectionAttributes(scope: constructs.Construct, id: string, attrs: ConnectionAttributes): IConnection { + public static fromConnectionArn(scope: constructs.Construct, id: string, connectionArn: string): IConnection { class Import extends cdk.Resource implements IConnection { - public readonly connectionName = attrs.connectionName; - public readonly connectionArn = Connection.buildConnectionArn(scope, attrs.connectionName); + public readonly connectionName = Connection.extractConnectionNameFromArn(connectionArn); + public readonly connectionArn = connectionArn; } return new Import(scope, id); } + /** + * Creates a Connection construct that represents an external connection. + * + * @param scope The scope creating construct (usually `this`). + * @param id The construct's id. + * @param connectionName name of external connection. + */ + public static fromConnectionName(scope: constructs.Construct, id: string, connectionName: string): IConnection { + class Import extends cdk.Resource implements IConnection { + public readonly connectionName = connectionName; + public readonly connectionArn = Connection.buildConnectionArn(scope, connectionName); + } + + return new Import(scope, id); + } + + /** + * Given an opaque (token) ARN, returns a CloudFormation expression that extracts the connection + * name from the ARN. + * + * Connection ARNs look like this: + * + * arn:aws:glue:region:account-id:connection/connection name + * + * @returns `FnSelect(1, FnSplit('/', arn))` + */ + private static extractConnectionNameFromArn(connectionArn: string) { + return cdk.Fn.select(1, cdk.Fn.split('/', connectionArn)); + } + private static buildConnectionArn(scope: constructs.Construct, connectionName: string) : string { return cdk.Stack.of(scope).formatArn({ service: 'glue', @@ -177,8 +195,8 @@ export class Connection extends cdk.Resource implements IConnection { const connectionResource = new CfnConnection(this, 'Resource', { catalogId: cdk.Aws.ACCOUNT_ID, connectionInput: { - connectionProperties: props.connectionProperties, - connectionType: props.connectionType.name, + connectionProperties: props.properties, + connectionType: props.type.name, description: props.description, matchCriteria: props.matchCriteria, name: props.connectionName, diff --git a/packages/@aws-cdk/aws-glue/test/connection.test.ts b/packages/@aws-cdk/aws-glue/test/connection.test.ts index 03f63261a32b7..1933a5216ec8d 100644 --- a/packages/@aws-cdk/aws-glue/test/connection.test.ts +++ b/packages/@aws-cdk/aws-glue/test/connection.test.ts @@ -8,8 +8,8 @@ import * as glue from '../lib'; test('a connection with connection properties', () => { const stack = new cdk.Stack(); new glue.Connection(stack, 'Connection', { - connectionType: glue.ConnectionType.JDBC, - connectionProperties: { + type: glue.ConnectionType.JDBC, + properties: { JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', USERNAME: 'username', PASSWORD: 'password', @@ -39,7 +39,7 @@ test('a connection with a subnet and security group', () => { }); const securityGroup = ec2.SecurityGroup.fromSecurityGroupId(stack, 'securityGroup', 'sgId'); new glue.Connection(stack, 'Connection', { - connectionType: glue.ConnectionType.NETWORK, + type: glue.ConnectionType.NETWORK, securityGroups: [securityGroup], subnet, }); @@ -64,7 +64,7 @@ test('a connection with a name and description', () => { new glue.Connection(stack, 'Connection', { connectionName: 'name', description: 'description', - connectionType: glue.ConnectionType.NETWORK, + type: glue.ConnectionType.NETWORK, }); expect(stack).to(haveResource('AWS::Glue::Connection', { @@ -84,7 +84,7 @@ test('a connection with a custom type', () => { new glue.Connection(stack, 'Connection', { connectionName: 'name', description: 'description', - connectionType: new glue.ConnectionType('CUSTOM_TYPE'), + type: new glue.ConnectionType('CUSTOM_TYPE'), }); expect(stack).to(haveResource('AWS::Glue::Connection', { @@ -102,7 +102,7 @@ test('a connection with a custom type', () => { test('a connection with match criteria', () => { const stack = new cdk.Stack(); new glue.Connection(stack, 'Connection', { - connectionType: glue.ConnectionType.NETWORK, + type: glue.ConnectionType.NETWORK, matchCriteria: ['c1', 'c2'], }); @@ -117,16 +117,24 @@ test('a connection with match criteria', () => { })); }); -test('fromConnectionAttributes', () => { +test('fromConnectionName', () => { + const connectionName = 'name'; const stack = new cdk.Stack(); - const connection = glue.Connection.fromConnectionAttributes(stack, 'ImportedConnection', { - connectionName: 'name', - }); + const connection = glue.Connection.fromConnectionName(stack, 'ImportedConnection', connectionName); - strictEqual(connection.connectionName, 'name'); + strictEqual(connection.connectionName, connectionName); strictEqual(connection.connectionArn, stack.formatArn({ service: 'glue', resource: 'connection', - resourceName: 'name', + resourceName: connectionName, })); +}); + +test('fromConnectionArn', () => { + const connectionArn = 'arn:aws:glue:region:account-id:connection/name'; + const stack = new cdk.Stack(); + const connection = glue.Connection.fromConnectionArn(stack, 'ImportedConnection', connectionArn); + + strictEqual(connection.connectionName, 'name'); + strictEqual(connection.connectionArn, connectionArn); }); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.connection.ts b/packages/@aws-cdk/aws-glue/test/integ.connection.ts index 4174c660cc568..3d586f97c85c0 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.connection.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.connection.ts @@ -14,7 +14,7 @@ const sg = new ec2.SecurityGroup(stack, 'SecurityGroup', { // Network connection new glue.Connection(stack, 'NetworkConnection', { - connectionType: glue.ConnectionType.NETWORK, + type: glue.ConnectionType.NETWORK, subnet: vpc.privateSubnets[0], securityGroups: [sg], }); From 080220c3c8a6c25480628fa65ad497bc30fb07aa Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Mon, 15 Feb 2021 16:38:48 +0000 Subject: [PATCH 06/12] address extensibility comments - add addProperty escape hatch in Connection - Extract ConnectionOptopns out of ConnectionProps - implement MongoDBConnection as a subclass of Connection --- packages/@aws-cdk/aws-glue/lib/connection.ts | 46 ++++++---- packages/@aws-cdk/aws-glue/lib/index.ts | 1 + .../aws-glue/lib/mongodb-connection.ts | 83 +++++++++++++++++++ .../@aws-cdk/aws-glue/test/connection.test.ts | 20 +++++ .../aws-glue/test/mongodb-connection.test.ts | 28 +++++++ 5 files changed, 164 insertions(+), 14 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts create mode 100644 packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts diff --git a/packages/@aws-cdk/aws-glue/lib/connection.ts b/packages/@aws-cdk/aws-glue/lib/connection.ts index 9e67c0fbeedc9..6001e2acc9024 100644 --- a/packages/@aws-cdk/aws-glue/lib/connection.ts +++ b/packages/@aws-cdk/aws-glue/lib/connection.ts @@ -66,9 +66,9 @@ export interface IConnection extends cdk.IResource { } /** - * Construction properties for {@link Connection} + * Base Connection Options */ -export interface ConnectionProps { +export interface ConnectionOptions { /** * The name of the connection * @default cloudformation generated name @@ -81,17 +81,6 @@ export interface ConnectionProps { */ readonly description?: string; - /** - * The type of the connection - */ - readonly type: ConnectionType; - - /** - * Key-Value pairs that define parameters for the connection. - * @default empty properties - */ - readonly properties?: { [key: string]: string }; - /** * A list of criteria that can be used in selecting this connection. * This is useful for filtering the results of https://awscli.amazonaws.com/v2/documentation/api/latest/reference/glue/get-connections.html @@ -112,6 +101,22 @@ export interface ConnectionProps { readonly subnet?: ec2.ISubnet; } +/** + * Construction properties for {@link Connection} + */ +export interface ConnectionProps extends ConnectionOptions { + /** + * The type of the connection + */ + readonly type: ConnectionType; + + /** + * Key-Value pairs that define parameters for the connection. + * @default empty properties + */ + readonly properties?: { [key: string]: string }; +} + /** * An AWS Glue connection to a data source. */ @@ -181,11 +186,15 @@ export class Connection extends cdk.Resource implements IConnection { */ public readonly connectionName: string; + private readonly properties: {[key: string]: string}; + constructor(scope: constructs.Construct, id: string, props: ConnectionProps) { super(scope, id, { physicalName: props.connectionName, }); + this.properties = props.properties || {}; + const physicalConnectionRequirements = props.subnet || props.securityGroups ? { availabilityZone: props.subnet ? props.subnet.availabilityZone : undefined, subnetId: props.subnet ? props.subnet.subnetId : undefined, @@ -195,7 +204,7 @@ export class Connection extends cdk.Resource implements IConnection { const connectionResource = new CfnConnection(this, 'Resource', { catalogId: cdk.Aws.ACCOUNT_ID, connectionInput: { - connectionProperties: props.properties, + connectionProperties: cdk.Lazy.any({ produce: () => Object.keys(this.properties).length > 0 ? this.properties : undefined }), connectionType: props.type.name, description: props.description, matchCriteria: props.matchCriteria, @@ -208,4 +217,13 @@ export class Connection extends cdk.Resource implements IConnection { this.connectionArn = Connection.buildConnectionArn(this, resourceName); this.connectionName = resourceName; } + + /** + * Add additional connection parameters + * @param key parameter key + * @param value parameter value + */ + public addProperty(key: string, value: string): void { + this.properties[key] = value; + } } diff --git a/packages/@aws-cdk/aws-glue/lib/index.ts b/packages/@aws-cdk/aws-glue/lib/index.ts index 4880b3b7136b8..2cc70997a55b3 100644 --- a/packages/@aws-cdk/aws-glue/lib/index.ts +++ b/packages/@aws-cdk/aws-glue/lib/index.ts @@ -2,6 +2,7 @@ export * from './glue.generated'; export * from './connection'; +export * from './mongodb-connection'; export * from './database'; export * from './schema'; export * from './data-format'; diff --git a/packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts b/packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts new file mode 100644 index 0000000000000..fe56ccb8eeda3 --- /dev/null +++ b/packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts @@ -0,0 +1,83 @@ +import * as constructs from 'constructs'; +import { Connection, ConnectionOptions, ConnectionType } from './connection'; + +/** + * Construction properties for {@link MongoDBConnection} + * + * @see https://docs.aws.amazon.com/glue/latest/dg/connection-mongodb.html + */ +export interface MongoDBConnectionProps extends ConnectionOptions { + /** + * The MongoDB database to read from. + */ + readonly database: string; + + /** + * The MongoDB collection to read from. + */ + readonly collection: string; + + /** + * If true, then AWS Glue initiates an SSL connection. + * + * @default false + */ + readonly ssl?: boolean; + + /** + * If true and ssl is true, then AWS Glue performs a domain match check. + * + * @default true + */ + readonly sslDomainMatch?: boolean; + + /** + * The number of documents to return per batch, used within the cursor of internal batches. + * + * @default unknown + */ + readonly batchSize?: number; + + + /** + * The class name of the partitioner for reading input data from MongoDB. The connector provides the following partitioners: + * - MongoDefaultPartitioner (default) + * - MongoSamplePartitioner (Requires MongoDB 3.2 or later) + * - MongoShardedPartitioner + * - MongoSplitVectorPartitioner + * - MongoPaginateByCountPartitioner + * - MongoPaginateBySizePartitioner + * + * @default MongoDefaultPartitioner + */ + readonly partitioner?: string; +} + +/** + * An AWS Glue connection to a MongoDB source. + */ +export class MongoDBConnection extends Connection { + + constructor(scope: constructs.Construct, id: string, props: MongoDBConnectionProps) { + super(scope, id, { + ...props, + type: ConnectionType.MONGODB, + properties: { + database: props.database, + collection: props.collection, + }, + }); + if (props.ssl !== undefined) { + this.addProperty('ssl', props.ssl.toString()); + } + if (props.sslDomainMatch !== undefined) { + this.addProperty('ssl.domain_match', props.sslDomainMatch.toString()); + } + if (props.batchSize !== undefined) { + this.addProperty('batchSize', props.batchSize.toString()); + } + if (props.partitioner !== undefined) { + this.addProperty('partitioner', props.partitioner); + } + } +} diff --git a/packages/@aws-cdk/aws-glue/test/connection.test.ts b/packages/@aws-cdk/aws-glue/test/connection.test.ts index 1933a5216ec8d..30e0c9324d89e 100644 --- a/packages/@aws-cdk/aws-glue/test/connection.test.ts +++ b/packages/@aws-cdk/aws-glue/test/connection.test.ts @@ -117,6 +117,26 @@ test('a connection with match criteria', () => { })); }); +test('addProperty', () => { + const stack = new cdk.Stack(); + const connection = new glue.Connection(stack, 'Connection', { + type: glue.ConnectionType.NETWORK, + }); + connection.addProperty('SomeKey', 'SomeValue'); + + expect(stack).to(haveResource('AWS::Glue::Connection', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + ConnectionInput: { + ConnectionType: 'NETWORK', + ConnectionProperties: { + SomeKey: 'SomeValue', + }, + }, + })); +}); + test('fromConnectionName', () => { const connectionName = 'name'; const stack = new cdk.Stack(); diff --git a/packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts b/packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts new file mode 100644 index 0000000000000..b0fdf416b254f --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts @@ -0,0 +1,28 @@ +import { expect, haveResource } from '@aws-cdk/assert'; +import * as cdk from '@aws-cdk/core'; +import * as glue from '../lib/index'; + +test('a mongodb connection with required properties only', () => { + const stack = new cdk.Stack(); + new glue.MongoDBConnection(stack, 'Connection', { + database: 'database', + collection: 'collection', + batchSize: 100, + ssl: true, + }); + + expect(stack).to(haveResource('AWS::Glue::Connection', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + ConnectionInput: { + ConnectionProperties: { + database: 'database', + collection: 'collection', + batchSize: '100', + ssl: 'true', + }, + ConnectionType: 'MONGODB', + }, + })); +}); \ No newline at end of file From 4a83cd3dab3e94b9948e0450b1dc7866c8c2e271 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Mon, 15 Feb 2021 18:46:10 +0000 Subject: [PATCH 07/12] address comments --- packages/@aws-cdk/aws-glue/README.md | 35 +++++++++++++++++++ packages/@aws-cdk/aws-glue/lib/connection.ts | 30 +++++----------- .../aws-glue/lib/mongodb-connection.ts | 13 ++++--- .../@aws-cdk/aws-glue/test/connection.test.ts | 25 +++++++------ .../aws-glue/test/mongodb-connection.test.ts | 4 +-- 5 files changed, 66 insertions(+), 41 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index 9c0f47b2ea018..c3226b2186167 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -35,6 +35,41 @@ new glue.Connection(stack, 'MyConnection', { }); ``` +If you need to use a connection type that doesn't exist as a static member on `ConnectionType`, you can instantiate a `ConnectionType` object, e.g: `new glue.ConnectionType('NEW_TYPE')`. + +Different connection types require different connection properties which can be set via `ConnectionProps.properties`. + +Subclasses of `Connection` are implemented to provide a connection type tailored props e.g. `MongoDBSourceConnection` which can be used as follows + +```ts +new glue.MongoDBSourceConnection(stack, 'Connection', { + database: 'database', + collection: 'collection', +}); +``` + +if a specific connection type construct props does not expose all properties, additional properties can still be set as follows + +```ts +new glue.MongoDBSourceConnection(stack, 'Connection', { + database: 'database', + collection: 'collection', + properties: { + key: 'value', + }, +}); +``` + +or + +```ts +const connection = new glue.MongoDBSourceConnection(stack, 'Connection', { + database: 'database', + collection: 'collection', +}); +connection.addProperty('key', 'value'); +``` + See [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations. ## Database diff --git a/packages/@aws-cdk/aws-glue/lib/connection.ts b/packages/@aws-cdk/aws-glue/lib/connection.ts index 6001e2acc9024..36e2d7abbb734 100644 --- a/packages/@aws-cdk/aws-glue/lib/connection.ts +++ b/packages/@aws-cdk/aws-glue/lib/connection.ts @@ -81,6 +81,12 @@ export interface ConnectionOptions { */ readonly description?: string; + /** + * Key-Value pairs that define parameters for the connection. + * @default empty properties + */ + readonly properties?: { [key: string]: string }; + /** * A list of criteria that can be used in selecting this connection. * This is useful for filtering the results of https://awscli.amazonaws.com/v2/documentation/api/latest/reference/glue/get-connections.html @@ -109,12 +115,6 @@ export interface ConnectionProps extends ConnectionOptions { * The type of the connection */ readonly type: ConnectionType; - - /** - * Key-Value pairs that define parameters for the connection. - * @default empty properties - */ - readonly properties?: { [key: string]: string }; } /** @@ -131,7 +131,7 @@ export class Connection extends cdk.Resource implements IConnection { */ public static fromConnectionArn(scope: constructs.Construct, id: string, connectionArn: string): IConnection { class Import extends cdk.Resource implements IConnection { - public readonly connectionName = Connection.extractConnectionNameFromArn(connectionArn); + public readonly connectionName = cdk.Arn.extractResourceName(connectionArn, 'connection'); public readonly connectionArn = connectionArn; } @@ -154,20 +154,6 @@ export class Connection extends cdk.Resource implements IConnection { return new Import(scope, id); } - /** - * Given an opaque (token) ARN, returns a CloudFormation expression that extracts the connection - * name from the ARN. - * - * Connection ARNs look like this: - * - * arn:aws:glue:region:account-id:connection/connection name - * - * @returns `FnSelect(1, FnSplit('/', arn))` - */ - private static extractConnectionNameFromArn(connectionArn: string) { - return cdk.Fn.select(1, cdk.Fn.split('/', connectionArn)); - } - private static buildConnectionArn(scope: constructs.Construct, connectionName: string) : string { return cdk.Stack.of(scope).formatArn({ service: 'glue', @@ -202,7 +188,7 @@ export class Connection extends cdk.Resource implements IConnection { } : undefined; const connectionResource = new CfnConnection(this, 'Resource', { - catalogId: cdk.Aws.ACCOUNT_ID, + catalogId: cdk.Stack.of(this).account, connectionInput: { connectionProperties: cdk.Lazy.any({ produce: () => Object.keys(this.properties).length > 0 ? this.properties : undefined }), connectionType: props.type.name, diff --git a/packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts b/packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts index fe56ccb8eeda3..8804867eb0386 100644 --- a/packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts +++ b/packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts @@ -2,11 +2,12 @@ import * as constructs from 'constructs'; import { Connection, ConnectionOptions, ConnectionType } from './connection'; /** - * Construction properties for {@link MongoDBConnection} + * Construction properties for {@link MongoDBSourceConnection} * * @see https://docs.aws.amazon.com/glue/latest/dg/connection-mongodb.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-connect.html#aws-glue-programming-etl-connect-mongodb */ -export interface MongoDBConnectionProps extends ConnectionOptions { +export interface MongoDBSourceConnectionProps extends ConnectionOptions { /** * The MongoDB database to read from. */ @@ -55,14 +56,18 @@ export interface MongoDBConnectionProps extends ConnectionOptions { /** * An AWS Glue connection to a MongoDB source. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/connection-mongodb.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-connect.html#aws-glue-programming-etl-connect-mongodb */ -export class MongoDBConnection extends Connection { +export class MongoDBSourceConnection extends Connection { - constructor(scope: constructs.Construct, id: string, props: MongoDBConnectionProps) { + constructor(scope: constructs.Construct, id: string, props: MongoDBSourceConnectionProps) { super(scope, id, { ...props, type: ConnectionType.MONGODB, properties: { + ...props.properties, database: props.database, collection: props.collection, }, diff --git a/packages/@aws-cdk/aws-glue/test/connection.test.ts b/packages/@aws-cdk/aws-glue/test/connection.test.ts index 30e0c9324d89e..5846bf161fa69 100644 --- a/packages/@aws-cdk/aws-glue/test/connection.test.ts +++ b/packages/@aws-cdk/aws-glue/test/connection.test.ts @@ -1,5 +1,4 @@ -import { strictEqual } from 'assert'; -import { expect, haveResource } from '@aws-cdk/assert'; +import * as cdkassert from '@aws-cdk/assert'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as cdk from '@aws-cdk/core'; import '@aws-cdk/assert/jest'; @@ -16,7 +15,7 @@ test('a connection with connection properties', () => { }, }); - expect(stack).to(haveResource('AWS::Glue::Connection', { + cdkassert.expect(stack).to(cdkassert.haveResource('AWS::Glue::Connection', { CatalogId: { Ref: 'AWS::AccountId', }, @@ -44,7 +43,7 @@ test('a connection with a subnet and security group', () => { subnet, }); - expect(stack).to(haveResource('AWS::Glue::Connection', { + cdkassert.expect(stack).to(cdkassert.haveResource('AWS::Glue::Connection', { CatalogId: { Ref: 'AWS::AccountId', }, @@ -67,7 +66,7 @@ test('a connection with a name and description', () => { type: glue.ConnectionType.NETWORK, }); - expect(stack).to(haveResource('AWS::Glue::Connection', { + cdkassert.expect(stack).to(cdkassert.haveResource('AWS::Glue::Connection', { CatalogId: { Ref: 'AWS::AccountId', }, @@ -87,7 +86,7 @@ test('a connection with a custom type', () => { type: new glue.ConnectionType('CUSTOM_TYPE'), }); - expect(stack).to(haveResource('AWS::Glue::Connection', { + cdkassert.expect(stack).to(cdkassert.haveResource('AWS::Glue::Connection', { CatalogId: { Ref: 'AWS::AccountId', }, @@ -106,7 +105,7 @@ test('a connection with match criteria', () => { matchCriteria: ['c1', 'c2'], }); - expect(stack).to(haveResource('AWS::Glue::Connection', { + cdkassert.expect(stack).to(cdkassert.haveResource('AWS::Glue::Connection', { CatalogId: { Ref: 'AWS::AccountId', }, @@ -124,7 +123,7 @@ test('addProperty', () => { }); connection.addProperty('SomeKey', 'SomeValue'); - expect(stack).to(haveResource('AWS::Glue::Connection', { + cdkassert.expect(stack).to(cdkassert.haveResource('AWS::Glue::Connection', { CatalogId: { Ref: 'AWS::AccountId', }, @@ -142,8 +141,8 @@ test('fromConnectionName', () => { const stack = new cdk.Stack(); const connection = glue.Connection.fromConnectionName(stack, 'ImportedConnection', connectionName); - strictEqual(connection.connectionName, connectionName); - strictEqual(connection.connectionArn, stack.formatArn({ + expect(connection.connectionName).toEqual(connectionName); + expect(connection.connectionArn).toEqual(stack.formatArn({ service: 'glue', resource: 'connection', resourceName: connectionName, @@ -155,6 +154,6 @@ test('fromConnectionArn', () => { const stack = new cdk.Stack(); const connection = glue.Connection.fromConnectionArn(stack, 'ImportedConnection', connectionArn); - strictEqual(connection.connectionName, 'name'); - strictEqual(connection.connectionArn, connectionArn); -}); \ No newline at end of file + expect(connection.connectionName).toEqual('name'); + expect(connection.connectionArn).toEqual(connectionArn); +}); diff --git a/packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts b/packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts index b0fdf416b254f..64ec9b28818c9 100644 --- a/packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts +++ b/packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts @@ -4,7 +4,7 @@ import * as glue from '../lib/index'; test('a mongodb connection with required properties only', () => { const stack = new cdk.Stack(); - new glue.MongoDBConnection(stack, 'Connection', { + new glue.MongoDBSourceConnection(stack, 'Connection', { database: 'database', collection: 'collection', batchSize: 100, @@ -25,4 +25,4 @@ test('a mongodb connection with required properties only', () => { ConnectionType: 'MONGODB', }, })); -}); \ No newline at end of file +}); From c7f6b401d52e085fa084d1b99fbe8229c5b0caa3 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Tue, 16 Feb 2021 18:57:41 +0000 Subject: [PATCH 08/12] Update packages/@aws-cdk/aws-glue/README.md Co-authored-by: Eli Polonsky --- packages/@aws-cdk/aws-glue/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index ae597a35cf6dc..e8693f16602ac 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -39,7 +39,9 @@ If you need to use a connection type that doesn't exist as a static member on `C Different connection types require different connection properties which can be set via `ConnectionProps.properties`. -Subclasses of `Connection` are implemented to provide a connection type tailored props e.g. `MongoDBSourceConnection` which can be used as follows +Concrete connection types are also available and provide a tailored, type-safe API for the specific connection options. + +### MongoDB ```ts new glue.MongoDBSourceConnection(stack, 'Connection', { From edbcc953839f71e0cbfaed60cb245df05d4a20f4 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Tue, 16 Feb 2021 18:57:51 +0000 Subject: [PATCH 09/12] Update packages/@aws-cdk/aws-glue/README.md Co-authored-by: Eli Polonsky --- packages/@aws-cdk/aws-glue/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index e8693f16602ac..b378454a67633 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -50,7 +50,7 @@ new glue.MongoDBSourceConnection(stack, 'Connection', { }); ``` -if a specific connection type construct props does not expose all properties, additional properties can still be set as follows +In case a specific connection type construct does not expose all properties, additional properties can still be set using the `properties` property. ```ts new glue.MongoDBSourceConnection(stack, 'Connection', { From aaf5d23f845f4b13e8135145934c3d1c9413ebd8 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Tue, 16 Feb 2021 18:58:19 +0000 Subject: [PATCH 10/12] Update packages/@aws-cdk/aws-glue/lib/connection.ts Co-authored-by: Eli Polonsky --- packages/@aws-cdk/aws-glue/lib/connection.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/@aws-cdk/aws-glue/lib/connection.ts b/packages/@aws-cdk/aws-glue/lib/connection.ts index 36e2d7abbb734..56cd115e4c6f9 100644 --- a/packages/@aws-cdk/aws-glue/lib/connection.ts +++ b/packages/@aws-cdk/aws-glue/lib/connection.ts @@ -84,6 +84,7 @@ export interface ConnectionOptions { /** * Key-Value pairs that define parameters for the connection. * @default empty properties + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-connect.html */ readonly properties?: { [key: string]: string }; From 250c5c53ffae60cdcc1effa3891ea6bc9923642e Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Wed, 17 Feb 2021 11:25:52 +0000 Subject: [PATCH 11/12] drop mongodb connection and address comments --- packages/@aws-cdk/aws-glue/README.md | 35 -------- packages/@aws-cdk/aws-glue/lib/index.ts | 1 - .../aws-glue/lib/mongodb-connection.ts | 88 ------------------- .../aws-glue/test/mongodb-connection.test.ts | 28 ------ 4 files changed, 152 deletions(-) delete mode 100644 packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts delete mode 100644 packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index b378454a67633..d2fac0d989a4a 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -37,41 +37,6 @@ new glue.Connection(stack, 'MyConnection', { If you need to use a connection type that doesn't exist as a static member on `ConnectionType`, you can instantiate a `ConnectionType` object, e.g: `new glue.ConnectionType('NEW_TYPE')`. -Different connection types require different connection properties which can be set via `ConnectionProps.properties`. - -Concrete connection types are also available and provide a tailored, type-safe API for the specific connection options. - -### MongoDB - -```ts -new glue.MongoDBSourceConnection(stack, 'Connection', { - database: 'database', - collection: 'collection', -}); -``` - -In case a specific connection type construct does not expose all properties, additional properties can still be set using the `properties` property. - -```ts -new glue.MongoDBSourceConnection(stack, 'Connection', { - database: 'database', - collection: 'collection', - properties: { - key: 'value', - }, -}); -``` - -or - -```ts -const connection = new glue.MongoDBSourceConnection(stack, 'Connection', { - database: 'database', - collection: 'collection', -}); -connection.addProperty('key', 'value'); -``` - See [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations. ## Database diff --git a/packages/@aws-cdk/aws-glue/lib/index.ts b/packages/@aws-cdk/aws-glue/lib/index.ts index 9784be60f3f36..a3dfa85b3be71 100644 --- a/packages/@aws-cdk/aws-glue/lib/index.ts +++ b/packages/@aws-cdk/aws-glue/lib/index.ts @@ -2,7 +2,6 @@ export * from './glue.generated'; export * from './connection'; -export * from './mongodb-connection'; export * from './data-format'; export * from './database'; export * from './schema'; diff --git a/packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts b/packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts deleted file mode 100644 index 8804867eb0386..0000000000000 --- a/packages/@aws-cdk/aws-glue/lib/mongodb-connection.ts +++ /dev/null @@ -1,88 +0,0 @@ -import * as constructs from 'constructs'; -import { Connection, ConnectionOptions, ConnectionType } from './connection'; - -/** - * Construction properties for {@link MongoDBSourceConnection} - * - * @see https://docs.aws.amazon.com/glue/latest/dg/connection-mongodb.html - * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-connect.html#aws-glue-programming-etl-connect-mongodb - */ -export interface MongoDBSourceConnectionProps extends ConnectionOptions { - /** - * The MongoDB database to read from. - */ - readonly database: string; - - /** - * The MongoDB collection to read from. - */ - readonly collection: string; - - /** - * If true, then AWS Glue initiates an SSL connection. - * - * @default false - */ - readonly ssl?: boolean; - - /** - * If true and ssl is true, then AWS Glue performs a domain match check. - * - * @default true - */ - readonly sslDomainMatch?: boolean; - - /** - * The number of documents to return per batch, used within the cursor of internal batches. - * - * @default unknown - */ - readonly batchSize?: number; - - - /** - * The class name of the partitioner for reading input data from MongoDB. The connector provides the following partitioners: - * - MongoDefaultPartitioner (default) - * - MongoSamplePartitioner (Requires MongoDB 3.2 or later) - * - MongoShardedPartitioner - * - MongoSplitVectorPartitioner - * - MongoPaginateByCountPartitioner - * - MongoPaginateBySizePartitioner - * - * @default MongoDefaultPartitioner - */ - readonly partitioner?: string; -} - -/** - * An AWS Glue connection to a MongoDB source. - * - * @see https://docs.aws.amazon.com/glue/latest/dg/connection-mongodb.html - * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-connect.html#aws-glue-programming-etl-connect-mongodb - */ -export class MongoDBSourceConnection extends Connection { - - constructor(scope: constructs.Construct, id: string, props: MongoDBSourceConnectionProps) { - super(scope, id, { - ...props, - type: ConnectionType.MONGODB, - properties: { - ...props.properties, - database: props.database, - collection: props.collection, - }, - }); - if (props.ssl !== undefined) { - this.addProperty('ssl', props.ssl.toString()); - } - if (props.sslDomainMatch !== undefined) { - this.addProperty('ssl.domain_match', props.sslDomainMatch.toString()); - } - if (props.batchSize !== undefined) { - this.addProperty('batchSize', props.batchSize.toString()); - } - if (props.partitioner !== undefined) { - this.addProperty('partitioner', props.partitioner); - } - } -} diff --git a/packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts b/packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts deleted file mode 100644 index 64ec9b28818c9..0000000000000 --- a/packages/@aws-cdk/aws-glue/test/mongodb-connection.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { expect, haveResource } from '@aws-cdk/assert'; -import * as cdk from '@aws-cdk/core'; -import * as glue from '../lib/index'; - -test('a mongodb connection with required properties only', () => { - const stack = new cdk.Stack(); - new glue.MongoDBSourceConnection(stack, 'Connection', { - database: 'database', - collection: 'collection', - batchSize: 100, - ssl: true, - }); - - expect(stack).to(haveResource('AWS::Glue::Connection', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - ConnectionInput: { - ConnectionProperties: { - database: 'database', - collection: 'collection', - batchSize: '100', - ssl: 'true', - }, - ConnectionType: 'MONGODB', - }, - })); -}); From bfe5867ede8556368a4e02a3d3ee6858e40a2248 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Wed, 17 Feb 2021 13:47:09 +0000 Subject: [PATCH 12/12] address README comments --- packages/@aws-cdk/aws-glue/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index d2fac0d989a4a..20e08d7c14e31 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -25,12 +25,14 @@ This module is part of the [AWS Cloud Development Kit](https://github.com/aws/aw ## Connection -A `Connection` allows Glue jobs, crawlers and development endpoints to access certain types of data stores. +A `Connection` allows Glue jobs, crawlers and development endpoints to access certain types of data stores. For example, to create a network connection to connect to a data source within a VPC: ```ts new glue.Connection(stack, 'MyConnection', { connectionType: glue.ConnectionTypes.NETWORK, + // The security groups granting AWS Glue inbound access to the data source within the VPC securityGroups: [securityGroup], + // The VPC subnet which contains the data source subnet, }); ```