From 1b0ffa52070ec3205a926d68de0b0b4deef8d88f Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Fri, 2 Aug 2024 20:55:19 +0000 Subject: [PATCH 01/18] initial commit --- packages/@aws-cdk/aws-glue-alpha/lib/index.ts | 1 + .../@aws-cdk/aws-glue-alpha/lib/workflow.ts | 549 ++++++++++++++++++ .../aws-glue-alpha/test/workflow.test.ts | 156 +++++ 3 files changed, 706 insertions(+) create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/index.ts b/packages/@aws-cdk/aws-glue-alpha/lib/index.ts index 1b9514c14625e..cd7dee8e635dd 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/index.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/index.ts @@ -14,3 +14,4 @@ export * from './security-configuration'; export * from './storage-parameter'; export * from './table-base'; export * from './table-deprecated'; +export * from './workflow'; diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts new file mode 100644 index 0000000000000..48d8a398b0b0c --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts @@ -0,0 +1,549 @@ +import * as cdk from 'aws-cdk-lib'; +import { IResource, Resource } from 'aws-cdk-lib'; +import { Schedule } from 'aws-cdk-lib/aws-events'; +import { CfnCrawler, CfnTrigger, CfnWorkflow } from 'aws-cdk-lib/aws-glue'; +import { Construct } from 'constructs'; +import { IJob } from './job'; +import { ISecurityConfiguration } from './security-configuration'; + +/** + * The condition to be evaluated. + */ +export enum TriggerPredicateCondition { + /** + * Defines when all predicates are met. + */ + AND = 'ALL', + + /** + * Defines when any predicate is met. + */ + OR = 'ANY', +} + +/** + * The state of the predicate. + */ +export enum PredicateState { + /** + * The predicate is in a state of SUCCEEDED. + */ + SUCCEEDED = 'SUCCEEDED', + + /** + * The predicate is in a state of STOPPED. + */ + STOPPED = 'STOPPED', + + /** + * The predicate is in a state of TIMEOUT. + */ + TIMEOUT = 'TIMEOUT', + + /** + * The predicate is in a state of FAILED. + */ + FAILED = 'FAILED', +} + +/** + * The job predicates to be evaluated. + */ +export interface JobPredicate { + /** + * The job to be evaluated. + */ + readonly job: IJob; + + /** + * The state of the predicate. + */ + readonly state: PredicateState; +} + +/** + * The crawler predicates to be evaluated. + */ +export interface CrawlerPredicate { + /** + * The crawler to be evaluated. + */ + readonly crawler: CfnCrawler; + + /** + * The state of the predicate. + */ + readonly state: PredicateState; +} + +/** + * The action to be executed when the trigger fires. + */ +export interface Action { + /** + * The job to be executed, either this or `crawler` must be specified. + * + * @default - A job will not be executed. + */ + readonly job?: IJob; + + /** + * The crawler to be executed, either this or `job` must be specified. + * + * @default - A crawler will not be executed. + */ + readonly crawler?: CfnCrawler; + + /** + * After a job run starts, the `Duration` to wait before glue will trigger a CloudWatch event. + * + * @default - A notification will not be sent. + */ + readonly delayCloudwatchEvent?: cdk.Duration; + + /** + * The job arguments specifically for this run. + * + * @default - No arguments. + */ + readonly arguments?: {[key: string]: string}; + + /** + * A Security Configuration to be used for the action. + * + * @default - No security configuration. + */ + readonly securityConfiguration?: ISecurityConfiguration; + + /** + * The timeout for a job run. + * + * @default - No timeout. + */ + readonly timeout?: cdk.Duration; +} + +/** + * Properties for defining a new trigger to be assgined to a workflow. + */ +export interface TriggerProps { + /** + * The name of the trigger. + * + * @default - A CloudFormation generated name. + */ + readonly triggerName?: string; + + /** + * The description of the trigger. + * + * @default - No description. + */ + readonly description?: string; + + /** + * Whether this trigger is enabled. + * + * @default true + */ + readonly enabled?: boolean; + + /** + * The actions to be executed when this trigger fires. + */ + readonly actions: Action[]; +} + +/** + * Properties for defining a new schedule trigger, to be assigned to a workflow. + */ +export interface ScheduleTriggerProps extends TriggerProps { + /** + * The schedule for this trigger. + */ + readonly schedule: Schedule; +} + +/** + * Properties for defining a new notification trigger, to be assigned to a workflow. + */ +export interface NotificationTriggerProps extends TriggerProps { + /** + * The number of events that should be recieved from Amazon EventBridge before the EventBridge trigger is fired. + * + * @default - 1 + */ + readonly batchSize?: number; + + /** + * The Window duration, after which the EventBridge trigger fires. + * + * @default - 900 seconds + */ + readonly batchWindow?: cdk.Duration; +} + +/** + * Properties for defining a new conditional trigger, to be assigned to a workflow. + */ +export interface ConditionalTriggerProps extends TriggerProps { + /** + * The condition to be evaluated. + * + * @default - TriggerPredicateCondition.AND + */ + readonly predicateCondition?: TriggerPredicateCondition; + + /** + * The job predicates to be evaluated. + * + * @default - No job predicates. + */ + readonly jobPredicates?: JobPredicate[]; + + /** + * The crawler predicates to be evaluated. + * + * @default - No crawler predicates. + */ + readonly crawlerPredicates?: CrawlerPredicate[]; +} + +/** + * Properties representing either a new or imported workflow. + */ +export interface IWorkflow extends IResource { + /** + * The ARN of the workflow. + * + * @attribute + */ + readonly workflowArn: string; + + /** + * The name of the workflow. + * + * @attribute + */ + readonly workflowName: string; +} + +/** + * Properties for defining a new workflow. + */ +export interface WorkflowProps { + /** + * The name of the workflow. + * + * @default - A CloudFormation generated name. + */ + readonly workflowName?: string; + + /** + * A description of the workflow. + * + * @default - No description. + */ + readonly description?: string; + + /** + * The maximum concurrent runs allowed for the workflow. + * + * @default - No limit. + */ + readonly maxConcurrentRuns?: number; + + /** + * The default run properties for the workflow. + * + * @default - No default run properties. + */ + readonly defaultRunProperties?: {[key: string]: string}; +} + +abstract class WorkflowBase extends Resource implements IWorkflow { + public abstract readonly workflowArn: string; + public abstract readonly workflowName: string; + + private readonly triggers: CfnTrigger[] = []; + + /** + * Adds a trigger to the workflow to run on demand. + */ + public addOnDemandTrigger( + scope: Construct, + id: string, + props: TriggerProps, + ): void { + this.triggers.push( + new CfnTrigger(scope, id, { + type: 'ON_DEMAND', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)), + }), + ); + } + + /** + * Adds a trigger to the workflow to run on a daily schedule. + */ + public addDailyScheduleTrigger( + scope: Construct, + id: string, + props: TriggerProps, + ): void { + this.triggers.push( + new CfnTrigger(scope, id, { + type: 'SCHEDULED', + schedule: 'cron(0 0 * * ? *)', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)), + }), + ); + } + + /** + * Adds a trigger to the workflow to run on a weekly schedule. + */ + public addWeeklyScheduleTrigger( + scope: Construct, + id: string, + props: TriggerProps, + ): void { + this.triggers.push( + new CfnTrigger(scope, id, { + type: 'SCHEDULED', + schedule: 'cron(0 0 ? * SUN *)', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)), + }), + ); + } + + /** + * Adds a trigger to the workflow to run on a monthly schedule. + */ + public addMonthlyScheduleTrigger( + scope: Construct, + id: string, + props: TriggerProps, + ): void { + this.triggers.push( + new CfnTrigger(scope, id, { + type: 'SCHEDULED', + schedule: 'cron(0 0 1 * ? *)', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)) ?? [], + }), + ); + } + + /** + * Adds a trigger to the workflow to run on a custom schedule. + */ + public addCustomScheduleTrigger( + scope: Construct, + id: string, + props: ScheduleTriggerProps, + ): void { + this.triggers.push( + new CfnTrigger(scope, id, { + type: 'SCHEDULED', + schedule: props.schedule.expressionString, + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)) ?? [], + }), + ); + } + + /** + * Adds a trigger to the workflow to run on a notification event. + */ + public addNotifyEventTrigger( + scope: Construct, + id: string, + props: NotificationTriggerProps, + ): void { + this.triggers.push( + new CfnTrigger(scope, id, { + type: 'CONDITIONAL', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)) ?? [], + eventBatchingCondition: { + batchSize: props.batchSize ?? 1, + batchWindow: props.batchWindow?.toSeconds() ?? 900, + }, + }), + ); + } + + /** + * Adds a trigger to the workflow to run on a conditional event. + */ + public addConditionalTrigger( + scope: Construct, + id: string, + props: ConditionalTriggerProps, + ): void { + this.validatePredicates(props.jobPredicates, props.crawlerPredicates); + const conditions: CfnTrigger.ConditionProperty[] = []; + + if (props.jobPredicates) { + conditions.push( + ...props.jobPredicates.map(predicate => ({ + logical: props.predicateCondition, + jobName: predicate.job.jobName, + state: predicate.state, + logicalOperator: 'EQUALS', + })), + ); + } + if (props.crawlerPredicates) { + conditions.push( + ...props.crawlerPredicates.map(predicate => ({ + logical: props.predicateCondition, + crawlerName: predicate.crawler.name ?? predicate.crawler.ref, + crawlState: predicate.state, + logicalOperator: 'EQUALS', + })), + ); + } + + this.triggers.push( + new CfnTrigger(scope, id, { + type: 'CONDITIONAL', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)), + predicate: { + logical: props.predicateCondition, + conditions: conditions.length > 0 ? conditions : undefined, + }, + }), + ); + } + + protected renderAction(action: Action): CfnTrigger.ActionProperty { + return { + jobName: action.job?.jobName, + crawlerName: action.crawler?.name ?? action.crawler?.ref, + arguments: action.arguments, + securityConfiguration: + action.securityConfiguration?.securityConfigurationName, + timeout: action.timeout?.toMinutes(), + notificationProperty: { + notifyDelayAfter: action.delayCloudwatchEvent?.toMinutes(), + }, + }; + } + + protected validatePredicates( + jobPredicates?: JobPredicate[], + crawlerPredicates?: CrawlerPredicate[], + ) { + if (!jobPredicates && !crawlerPredicates) { + throw new Error( + 'At least one job predicate or crawler predicate must be specified.', + ); + } + if (!jobPredicates?.length && !crawlerPredicates?.length) { + throw new Error( + 'At least one job predicate or crawler predicate must be specified.', + ); + } + } + + // Cloudformation cannot handle the a glue workflow being updated multiple times at once. + protected addDependentTrigger(): void { + if (this.triggers.length > 1) { + this.triggers[this.triggers.length - 1].addDependency( + this.triggers[this.triggers.length - 2], + ); + } + } +} + +/** + * A Glue workflow. + */ +export class Workflow extends WorkflowBase implements IWorkflow { + /** + * Import an existing workflow, using its ARN. + */ + public static fromWorkflowArn( + scope: Construct, + id: string, + workflowArn: string, + ): IWorkflow { + class Import extends WorkflowBase implements IWorkflow { + public readonly workflowArn = workflowArn; + public readonly workflowName = cdk.Arn.extractResourceName( + workflowArn, + 'workflow', + ); + } + + return new Import(scope, id); + } + + /** + * Import an existing workflow, using its name. + */ + public static fromWorkflowName( + scope: Construct, + id: string, + workflowName: string, + ): IWorkflow { + class Import extends WorkflowBase implements IWorkflow { + public readonly workflowArn = Workflow.buildWorkflowArn( + scope, + workflowName, + ); + public readonly workflowName = workflowName; + } + + return new Import(scope, id); + } + + private static buildWorkflowArn( + scope: Construct, + workflowName: string, + ): string { + return cdk.Stack.of(scope).formatArn({ + service: 'glue', + resource: 'workflow', + resourceName: workflowName, + }); + } + + /** + * Name of this workflow. + */ + public readonly workflowName: string; + + /** + * ARN of this workflow. + */ + public readonly workflowArn: string; + + constructor(scope: Construct, id: string, props?: WorkflowProps) { + super(scope, id); + + const resource = new CfnWorkflow(this, 'Resource', { + name: props?.workflowName, + description: props?.description, + maxConcurrentRuns: props?.maxConcurrentRuns, + defaultRunProperties: props?.defaultRunProperties, + }); + + const resourceName = this.getResourceNameAttribute(resource.ref); + this.workflowName = resourceName; + this.workflowArn = Workflow.buildWorkflowArn(scope, this.workflowName); + } +} diff --git a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts new file mode 100644 index 0000000000000..be0b3be9c57cc --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts @@ -0,0 +1,156 @@ +import * as cdk from 'aws-cdk-lib'; +import { Template } from 'aws-cdk-lib/assertions'; +import * as glueCfn from 'aws-cdk-lib/aws-glue'; +import * as glue from '../lib'; + +test('workflow', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + new glue.Workflow(stack, 'Workflow'); + + // THEN + Template.fromStack(stack).hasResource('AWS::Glue::Workflow', { + Name: 'Workflow', + }); +}); + +test('workflow with props', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + new glue.Workflow(stack, 'myWorkflow', { + workflowName: 'myWorkflow', + description: 'myDescription', + maxConcurrentRuns: 1, + defaultRunProperties: { + foo: 'bar', + key: 'value', + }, + }); + + // THEN + Template.fromStack(stack).hasResource('AWS::Glue::Workflow', { + Name: 'myWorkflow', + Description: 'myDescription', + MaxConcurrentRuns: 1, + DefaultRunProperties: { + foo: 'bar', + key: 'value', + }, + }); +}); + +describe('workflow with triggers', () => { + let stack: cdk.Stack; + let workflow: glue.Workflow; + let job: glue.Job; + let crawler: glueCfn.CfnCrawler; + let securityConfiguration: glue.SecurityConfiguration; + + beforeEach(() => { + stack = new cdk.Stack(); + workflow = new glue.Workflow(stack, 'myWorkflow'); + job = new glue.Job(stack, 'myJob', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script: glue.Code.fromAsset('myScript'), + }) + }); + crawler = new glueCfn.CfnCrawler(stack, 'myCrawler', { + role: 'myRole', + databaseName: 'myDatabase', + targets: { + s3Targets: [{ path: 'myPath' }] + } + }); + securityConfiguration = new glue.SecurityConfiguration(stack, 'mySecurityConfiguration'); + }); + + test('workflow with triggers', () => { + // WHEN + workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { + actions: [{ + job: job, + arguments: { + foo: 'bar', + key: 'value', + }, + delayCloudwatchEvent: cdk.Duration.seconds(5), + timeout: cdk.Duration.seconds(5 * 60), + }] + }); + + // THEN + Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Name: 'OnDemandTrigger', + Type: 'ON_DEMAND', + Actions: [{ + JobName: { 'Fn::GetAtt': ['myJobC2A9F6D7', 'Name'] }, + SecurityConfiguration: { 'Fn::GetAtt': ['mySecurityConfigurationC2A9F6D7', 'Name'] }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 300, + NotificationProperty: { + NotifyDelayAfter: 5, + }, + }], + WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + }); + }); +}); + +test('workflow with trigger that has both job and crawler as one action', () => { + // GIVEN + const stack = new cdk.Stack(); + const workflow = new glue.Workflow(stack, 'myWorkflow'); + const job = new glue.Job(stack, 'myJob', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script: glue.Code.fromAsset('myScript'), + }) + }); + const crawler = new glueCfn.CfnCrawler(stack, 'myCrawler', { + role: 'myRole', + databaseName: 'myDatabase', + targets: { + s3Targets: [{ path: 'myPath' }] + } + }); + + // WHEN + expect(() => workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { + actions: [{ + job: job, + crawler: crawler, + }] + }), + + // THEN + ).toThrow(/Only one of job or crawler can be specified in an action/); +}); + +test('workflow with trigger that has neither job nor crawler as one action', () => { + // GIVEN + const stack = new cdk.Stack(); + const workflow = new glue.Workflow(stack, 'myWorkflow'); + + // WHEN + expect(() => workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { + actions: [{ + arguments: { + foo: 'bar', + key: 'value', + }, + }] + }), + + // THEN + ).toThrow(/Either job or crawler must be specified in an action/); +}); \ No newline at end of file From 37cd92cb175225f85ecbe29fa9dbb1fa86eeb63f Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Fri, 2 Aug 2024 21:10:57 +0000 Subject: [PATCH 02/18] removing dependency check --- packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts index 48d8a398b0b0c..1c24176e14a49 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts @@ -458,15 +458,6 @@ abstract class WorkflowBase extends Resource implements IWorkflow { ); } } - - // Cloudformation cannot handle the a glue workflow being updated multiple times at once. - protected addDependentTrigger(): void { - if (this.triggers.length > 1) { - this.triggers[this.triggers.length - 1].addDependency( - this.triggers[this.triggers.length - 2], - ); - } - } } /** From cf8e089b3f45546cad089882b946ca4b22e72549 Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Fri, 2 Aug 2024 21:11:27 +0000 Subject: [PATCH 03/18] All tests should be added --- .../aws-glue-alpha/test/workflow.test.ts | 251 +++++++++++++++++- 1 file changed, 249 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts index be0b3be9c57cc..625c4c82f8378 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts @@ -1,5 +1,6 @@ import * as cdk from 'aws-cdk-lib'; import { Template } from 'aws-cdk-lib/assertions'; +import { Schedule } from 'aws-cdk-lib/aws-events'; import * as glueCfn from 'aws-cdk-lib/aws-glue'; import * as glue from '../lib'; @@ -70,7 +71,7 @@ describe('workflow with triggers', () => { securityConfiguration = new glue.SecurityConfiguration(stack, 'mySecurityConfiguration'); }); - test('workflow with triggers', () => { + test('onDemand', () => { // WHEN workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { actions: [{ @@ -81,6 +82,7 @@ describe('workflow with triggers', () => { }, delayCloudwatchEvent: cdk.Duration.seconds(5), timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, }] }); @@ -103,6 +105,251 @@ describe('workflow with triggers', () => { WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, }); }); + + test('dailySchedule', () => { + // WHEN + workflow.addDailyScheduleTrigger(stack, 'DailyScheduleTrigger', { + actions: [{ + crawler: crawler, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(5), + }] + }); + + // THEN + Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Name: 'DailyScheduleTrigger', + Type: 'SCHEDULED', + Schedule: 'cron(0 1 * * ? *)', + Actions: [{ + CrawlerName: { 'Fn::GetAtt': ['myCrawlerC2A9F6D7', 'Name'] }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 300, + }], + WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + }); + }); + + test('weeklySchedule', () => { + // WHEN + workflow.addWeeklyScheduleTrigger(stack, 'WeeklyScheduleTrigger', { + actions: [{ + job: job, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(5), + }] + }); + + // THEN + Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Name: 'WeeklyScheduleTrigger', + Type: 'SCHEDULED', + Schedule: 'cron(0 1 ? * MON *)', + Actions: [{ + JobName: { 'Fn::GetAtt': ['myJobC2A9F6D7', 'Name'] }, + SecurityConfiguration: { 'Fn::GetAtt': ['mySecurityConfigurationC2A9F6D7', 'Name'] }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 300, + NotificationProperty: { + NotifyDelayAfter: 5, + }, + }], + WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + }); + }); + + test('monthlySchedule', () => { + // WHEN + workflow.addMonthlyScheduleTrigger(stack, 'MonthlyScheduleTrigger', { + actions: [{ + crawler: crawler, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(5), + }] + }); + + // THEN + Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Name: 'MonthlyScheduleTrigger', + Type: 'SCHEDULED', + Schedule: 'cron(0 1 1 * ? *)', + Actions: [{ + CrawlerName: { 'Fn::GetAtt': ['myCrawlerC2A9F6D7', 'Name'] }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 300, + }], + WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + }); + }); + + test('customSchedule', () => { + // WHEN + workflow.addCustomScheduleTrigger(stack, 'CustomScheduleTrigger', { + schedule: Schedule.cron({ minute: '0', hour: '1', day: '1', month: 'JAN', year: '?', }), + actions: [{ + job: job, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(5), + }] + }); + + // THEN + Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Name: 'CustomScheduleTrigger', + Type: 'SCHEDULED', + Schedule: 'cron(0 1 1 JAN ? *)', + Actions: [{ + JobName: { 'Fn::GetAtt': ['myJobC2A9F6D7', 'Name'] }, + SecurityConfiguration: { 'Fn::GetAtt': ['mySecurityConfigurationC2A9F6D7', 'Name'] }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 300, + NotificationProperty: { + NotifyDelayAfter: 5, + }, + }], + WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + }); + }); + + test('notifyEvent', () => { + // WHEN + workflow.addNotifyEventTrigger(stack, 'OnDemandTrigger', { + batchSize: 50, + batchWindow: cdk.Duration.minutes(5), + actions: [{ + job: job, + arguments: { + foo: 'bar', + key: 'value', + }, + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(5), + timeout: cdk.Duration.seconds(5 * 60), + }], + }); + + // THEN + Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Name: 'OnDemandTrigger', + Type: 'CONDITIONAL', + Actions: [{ + JobName: { 'Fn::GetAtt': ['myJobC2A9F6D7', 'Name'] }, + Arguments: { + foo: 'bar', + key: 'value', + }, + NotificationProperty: { + NotifyDelayAfter: 0, + NotifyDelayAfterEvent: 'ON_FAILURE', + }, + }], + EventBatchingCondition: { + BatchSize: 50, + BatchWindow: 300, + }, + WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + }); + }); + + test('conditional', () => { + // WHEN + workflow.addConditionalTrigger(stack, 'ConditionalTrigger', { + actions: [{ + crawler: crawler, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(5), + }], + predicateCondition: glue.TriggerPredicateCondition.AND, + jobPredicates: [{ + job: job, + state: glue.PredicateState.SUCCEEDED, + }], + crawlerPredicates: [{ + crawler: crawler, + state: glue.PredicateState.SUCCEEDED, + }], + }); + + // THEN + Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Name: 'ConditionalTrigger', + Type: 'CONDITIONAL', + Actions: [{ + CrawlerName: { 'Fn::GetAtt': ['myCrawlerC2A9F6D7', 'Name'] }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 300, + }], + Predicate: { + Conditions: ['AND'], + Logical: { + JobName: { 'Fn::GetAtt': ['myJobC2A9F6D7', 'Name'] }, + State: 'SUCCEEDED', + }, + State: 'SUCCEEDED', + }, + WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + }); + }); + + test('conditional with no predicates', () => { + // WHEN + expect(() => + workflow.addConditionalTrigger(stack, 'ConditionalTrigger', { + actions: [{ + crawler: crawler, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(5), + }], + }), + + // THEN + ).toThrow(/At least one job predicate or crawler predicate must be specified./); + }); }); test('workflow with trigger that has both job and crawler as one action', () => { @@ -125,7 +372,7 @@ test('workflow with trigger that has both job and crawler as one action', () => }); // WHEN - expect(() => workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { + expect(() => workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { actions: [{ job: job, crawler: crawler, From a0f6083457a1eac538b07fc4a3c6301728fa5d9e Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Fri, 2 Aug 2024 21:14:02 +0000 Subject: [PATCH 04/18] eslint fix --- .../aws-glue-alpha/test/workflow.test.ts | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts index 625c4c82f8378..5684dbc89289e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts @@ -59,14 +59,14 @@ describe('workflow with triggers', () => { glueVersion: glue.GlueVersion.V2_0, pythonVersion: glue.PythonVersion.THREE, script: glue.Code.fromAsset('myScript'), - }) + }), }); crawler = new glueCfn.CfnCrawler(stack, 'myCrawler', { role: 'myRole', databaseName: 'myDatabase', targets: { - s3Targets: [{ path: 'myPath' }] - } + s3Targets: [{ path: 'myPath' }], + }, }); securityConfiguration = new glue.SecurityConfiguration(stack, 'mySecurityConfiguration'); }); @@ -83,9 +83,9 @@ describe('workflow with triggers', () => { delayCloudwatchEvent: cdk.Duration.seconds(5), timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, - }] + }], }); - + // THEN Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { Name: 'OnDemandTrigger', @@ -118,9 +118,9 @@ describe('workflow with triggers', () => { timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, delayCloudwatchEvent: cdk.Duration.seconds(5), - }] + }], }); - + // THEN Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { Name: 'DailyScheduleTrigger', @@ -150,9 +150,9 @@ describe('workflow with triggers', () => { timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, delayCloudwatchEvent: cdk.Duration.seconds(5), - }] + }], }); - + // THEN Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { Name: 'WeeklyScheduleTrigger', @@ -186,9 +186,9 @@ describe('workflow with triggers', () => { timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, delayCloudwatchEvent: cdk.Duration.seconds(5), - }] + }], }); - + // THEN Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { Name: 'MonthlyScheduleTrigger', @@ -209,7 +209,7 @@ describe('workflow with triggers', () => { test('customSchedule', () => { // WHEN workflow.addCustomScheduleTrigger(stack, 'CustomScheduleTrigger', { - schedule: Schedule.cron({ minute: '0', hour: '1', day: '1', month: 'JAN', year: '?', }), + schedule: Schedule.cron({ minute: '0', hour: '1', day: '1', month: 'JAN', year: '?' }), actions: [{ job: job, arguments: { @@ -219,9 +219,9 @@ describe('workflow with triggers', () => { timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, delayCloudwatchEvent: cdk.Duration.seconds(5), - }] + }], }); - + // THEN Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { Name: 'CustomScheduleTrigger', @@ -259,7 +259,7 @@ describe('workflow with triggers', () => { timeout: cdk.Duration.seconds(5 * 60), }], }); - + // THEN Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { Name: 'OnDemandTrigger', @@ -306,7 +306,7 @@ describe('workflow with triggers', () => { state: glue.PredicateState.SUCCEEDED, }], }); - + // THEN Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { Name: 'ConditionalTrigger', @@ -333,19 +333,19 @@ describe('workflow with triggers', () => { test('conditional with no predicates', () => { // WHEN - expect(() => + expect(() => workflow.addConditionalTrigger(stack, 'ConditionalTrigger', { - actions: [{ - crawler: crawler, - arguments: { - foo: 'bar', - key: 'value', - }, - timeout: cdk.Duration.seconds(5 * 60), - securityConfiguration: securityConfiguration, - delayCloudwatchEvent: cdk.Duration.seconds(5), - }], - }), + actions: [{ + crawler: crawler, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(5), + }], + }), // THEN ).toThrow(/At least one job predicate or crawler predicate must be specified./); @@ -361,14 +361,14 @@ test('workflow with trigger that has both job and crawler as one action', () => glueVersion: glue.GlueVersion.V2_0, pythonVersion: glue.PythonVersion.THREE, script: glue.Code.fromAsset('myScript'), - }) + }), }); const crawler = new glueCfn.CfnCrawler(stack, 'myCrawler', { role: 'myRole', databaseName: 'myDatabase', targets: { - s3Targets: [{ path: 'myPath' }] - } + s3Targets: [{ path: 'myPath' }], + }, }); // WHEN @@ -376,8 +376,8 @@ test('workflow with trigger that has both job and crawler as one action', () => actions: [{ job: job, crawler: crawler, - }] - }), + }], + }), // THEN ).toThrow(/Only one of job or crawler can be specified in an action/); @@ -389,14 +389,14 @@ test('workflow with trigger that has neither job nor crawler as one action', () const workflow = new glue.Workflow(stack, 'myWorkflow'); // WHEN - expect(() => workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { + expect(() => workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { actions: [{ arguments: { foo: 'bar', key: 'value', }, - }] - }), + }], + }), // THEN ).toThrow(/Either job or crawler must be specified in an action/); From 9913857020d789c5d2f460d65f0441ada4c10f29 Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Fri, 2 Aug 2024 21:28:45 +0000 Subject: [PATCH 05/18] incorrect syntax --- .../aws-glue-alpha/test/workflow.test.ts | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts index 5684dbc89289e..111d729754c85 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts @@ -2,6 +2,7 @@ import * as cdk from 'aws-cdk-lib'; import { Template } from 'aws-cdk-lib/assertions'; import { Schedule } from 'aws-cdk-lib/aws-events'; import * as glueCfn from 'aws-cdk-lib/aws-glue'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; import * as glue from '../lib'; test('workflow', () => { @@ -12,9 +13,7 @@ test('workflow', () => { new glue.Workflow(stack, 'Workflow'); // THEN - Template.fromStack(stack).hasResource('AWS::Glue::Workflow', { - Name: 'Workflow', - }); + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Workflow', {}); }); test('workflow with props', () => { @@ -33,7 +32,7 @@ test('workflow with props', () => { }); // THEN - Template.fromStack(stack).hasResource('AWS::Glue::Workflow', { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Workflow', { Name: 'myWorkflow', Description: 'myDescription', MaxConcurrentRuns: 1, @@ -58,7 +57,7 @@ describe('workflow with triggers', () => { executable: glue.JobExecutable.pythonEtl({ glueVersion: glue.GlueVersion.V2_0, pythonVersion: glue.PythonVersion.THREE, - script: glue.Code.fromAsset('myScript'), + script: glue.Code.fromBucket(Bucket.fromBucketName(stack, 'myBucket', 'my-bucket'), 'myKey'), }), }); crawler = new glueCfn.CfnCrawler(stack, 'myCrawler', { @@ -87,7 +86,7 @@ describe('workflow with triggers', () => { }); // THEN - Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { Name: 'OnDemandTrigger', Type: 'ON_DEMAND', Actions: [{ @@ -122,7 +121,7 @@ describe('workflow with triggers', () => { }); // THEN - Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { Name: 'DailyScheduleTrigger', Type: 'SCHEDULED', Schedule: 'cron(0 1 * * ? *)', @@ -154,7 +153,7 @@ describe('workflow with triggers', () => { }); // THEN - Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { Name: 'WeeklyScheduleTrigger', Type: 'SCHEDULED', Schedule: 'cron(0 1 ? * MON *)', @@ -190,7 +189,7 @@ describe('workflow with triggers', () => { }); // THEN - Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { Name: 'MonthlyScheduleTrigger', Type: 'SCHEDULED', Schedule: 'cron(0 1 1 * ? *)', @@ -223,7 +222,7 @@ describe('workflow with triggers', () => { }); // THEN - Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { Name: 'CustomScheduleTrigger', Type: 'SCHEDULED', Schedule: 'cron(0 1 1 JAN ? *)', @@ -261,7 +260,7 @@ describe('workflow with triggers', () => { }); // THEN - Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { Name: 'OnDemandTrigger', Type: 'CONDITIONAL', Actions: [{ @@ -308,7 +307,7 @@ describe('workflow with triggers', () => { }); // THEN - Template.fromStack(stack).hasResource('AWS::Glue::Trigger', { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { Name: 'ConditionalTrigger', Type: 'CONDITIONAL', Actions: [{ From 5f61b921db1c0ff63e05f7856e5f7191120815ea Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Fri, 2 Aug 2024 22:07:26 +0000 Subject: [PATCH 06/18] fixing code, based on unit test --- .../@aws-cdk/aws-glue-alpha/lib/workflow.ts | 26 +++- .../aws-glue-alpha/test/workflow.test.ts | 116 ++++++++++-------- 2 files changed, 87 insertions(+), 55 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts index 1c24176e14a49..92bcc9a09fef3 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts @@ -95,7 +95,7 @@ export interface Action { readonly crawler?: CfnCrawler; /** - * After a job run starts, the `Duration` to wait before glue will trigger a CloudWatch event. + * After a job run starts, the `Duration` to wait before glue will trigger a CloudWatch event, in minutes. * * @default - A notification will not be sent. */ @@ -116,7 +116,7 @@ export interface Action { readonly securityConfiguration?: ISecurityConfiguration; /** - * The timeout for a job run. + * The timeout for a job run, in minutes. * * @default - No timeout. */ @@ -277,6 +277,7 @@ abstract class WorkflowBase extends Resource implements IWorkflow { ): void { this.triggers.push( new CfnTrigger(scope, id, { + name: props.triggerName, type: 'ON_DEMAND', workflowName: this.workflowName, startOnCreation: props.enabled ?? true, @@ -295,8 +296,9 @@ abstract class WorkflowBase extends Resource implements IWorkflow { ): void { this.triggers.push( new CfnTrigger(scope, id, { + name: props.triggerName, type: 'SCHEDULED', - schedule: 'cron(0 0 * * ? *)', + schedule: 'cron(0 1 * * ? *)', workflowName: this.workflowName, startOnCreation: props.enabled ?? true, actions: props.actions?.map(action => this.renderAction(action)), @@ -314,8 +316,9 @@ abstract class WorkflowBase extends Resource implements IWorkflow { ): void { this.triggers.push( new CfnTrigger(scope, id, { + name: props.triggerName, type: 'SCHEDULED', - schedule: 'cron(0 0 ? * SUN *)', + schedule: 'cron(0 1 ? * MON *)', workflowName: this.workflowName, startOnCreation: props.enabled ?? true, actions: props.actions?.map(action => this.renderAction(action)), @@ -333,8 +336,9 @@ abstract class WorkflowBase extends Resource implements IWorkflow { ): void { this.triggers.push( new CfnTrigger(scope, id, { + name: props.triggerName, type: 'SCHEDULED', - schedule: 'cron(0 0 1 * ? *)', + schedule: 'cron(0 1 1 * ? *)', workflowName: this.workflowName, startOnCreation: props.enabled ?? true, actions: props.actions?.map(action => this.renderAction(action)) ?? [], @@ -352,6 +356,7 @@ abstract class WorkflowBase extends Resource implements IWorkflow { ): void { this.triggers.push( new CfnTrigger(scope, id, { + name: props.triggerName, type: 'SCHEDULED', schedule: props.schedule.expressionString, workflowName: this.workflowName, @@ -371,6 +376,7 @@ abstract class WorkflowBase extends Resource implements IWorkflow { ): void { this.triggers.push( new CfnTrigger(scope, id, { + name: props.triggerName, type: 'CONDITIONAL', workflowName: this.workflowName, startOnCreation: props.enabled ?? true, @@ -417,6 +423,7 @@ abstract class WorkflowBase extends Resource implements IWorkflow { this.triggers.push( new CfnTrigger(scope, id, { + name: props.triggerName, type: 'CONDITIONAL', workflowName: this.workflowName, startOnCreation: props.enabled ?? true, @@ -430,6 +437,15 @@ abstract class WorkflowBase extends Resource implements IWorkflow { } protected renderAction(action: Action): CfnTrigger.ActionProperty { + if (!action.job && !action.crawler) { + throw new Error('Either job or crawler must be specified in an action'); + } + if (action.job && action.crawler) { + throw new Error( + 'Only one of job or crawler can be specified in an action', + ); + } + return { jobName: action.job?.jobName, crawlerName: action.crawler?.name ?? action.crawler?.ref, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts index 111d729754c85..6d261c173e906 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts @@ -67,19 +67,24 @@ describe('workflow with triggers', () => { s3Targets: [{ path: 'myPath' }], }, }); - securityConfiguration = new glue.SecurityConfiguration(stack, 'mySecurityConfiguration'); + securityConfiguration = new glue.SecurityConfiguration(stack, 'mySecurityConfiguration', { + s3Encryption: { + mode: glue.S3EncryptionMode.S3_MANAGED, + }, + }); }); test('onDemand', () => { // WHEN workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { + triggerName: 'OnDemandTrigger', actions: [{ job: job, arguments: { foo: 'bar', key: 'value', }, - delayCloudwatchEvent: cdk.Duration.seconds(5), + delayCloudwatchEvent: cdk.Duration.seconds(60), timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, }], @@ -90,18 +95,18 @@ describe('workflow with triggers', () => { Name: 'OnDemandTrigger', Type: 'ON_DEMAND', Actions: [{ - JobName: { 'Fn::GetAtt': ['myJobC2A9F6D7', 'Name'] }, - SecurityConfiguration: { 'Fn::GetAtt': ['mySecurityConfigurationC2A9F6D7', 'Name'] }, + JobName: { Ref: 'myJob9A6589B3' }, + SecurityConfiguration: { Ref: 'mySecurityConfiguration58B0C573' }, Arguments: { foo: 'bar', key: 'value', }, - Timeout: 300, + Timeout: 5, NotificationProperty: { - NotifyDelayAfter: 5, + NotifyDelayAfter: 1, }, }], - WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + WorkflowName: { Ref: 'myWorkflow931F3265' }, }); }); @@ -116,24 +121,27 @@ describe('workflow with triggers', () => { }, timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, - delayCloudwatchEvent: cdk.Duration.seconds(5), + delayCloudwatchEvent: cdk.Duration.seconds(60), }], }); // THEN Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { - Name: 'DailyScheduleTrigger', Type: 'SCHEDULED', Schedule: 'cron(0 1 * * ? *)', Actions: [{ - CrawlerName: { 'Fn::GetAtt': ['myCrawlerC2A9F6D7', 'Name'] }, + CrawlerName: { Ref: 'myCrawler' }, Arguments: { foo: 'bar', key: 'value', }, - Timeout: 300, + Timeout: 5, + SecurityConfiguration: { Ref: 'mySecurityConfiguration58B0C573' }, + NotificationProperty: { + NotifyDelayAfter: 1, + }, }], - WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + WorkflowName: { Ref: 'myWorkflow931F3265' }, }); }); @@ -148,28 +156,27 @@ describe('workflow with triggers', () => { }, timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, - delayCloudwatchEvent: cdk.Duration.seconds(5), + delayCloudwatchEvent: cdk.Duration.seconds(60), }], }); // THEN Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { - Name: 'WeeklyScheduleTrigger', Type: 'SCHEDULED', Schedule: 'cron(0 1 ? * MON *)', Actions: [{ - JobName: { 'Fn::GetAtt': ['myJobC2A9F6D7', 'Name'] }, - SecurityConfiguration: { 'Fn::GetAtt': ['mySecurityConfigurationC2A9F6D7', 'Name'] }, + JobName: { Ref: 'myJob9A6589B3' }, + SecurityConfiguration: { Ref: 'mySecurityConfiguration58B0C573' }, Arguments: { foo: 'bar', key: 'value', }, - Timeout: 300, + Timeout: 5, NotificationProperty: { - NotifyDelayAfter: 5, + NotifyDelayAfter: 1, }, }], - WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + WorkflowName: { Ref: 'myWorkflow931F3265' }, }); }); @@ -184,24 +191,26 @@ describe('workflow with triggers', () => { }, timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, - delayCloudwatchEvent: cdk.Duration.seconds(5), + delayCloudwatchEvent: cdk.Duration.seconds(60), }], }); // THEN Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { - Name: 'MonthlyScheduleTrigger', Type: 'SCHEDULED', Schedule: 'cron(0 1 1 * ? *)', Actions: [{ - CrawlerName: { 'Fn::GetAtt': ['myCrawlerC2A9F6D7', 'Name'] }, + CrawlerName: { Ref: 'myCrawler' }, Arguments: { foo: 'bar', key: 'value', }, - Timeout: 300, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, }], - WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + WorkflowName: { Ref: 'myWorkflow931F3265' }, }); }); @@ -217,28 +226,27 @@ describe('workflow with triggers', () => { }, timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, - delayCloudwatchEvent: cdk.Duration.seconds(5), + delayCloudwatchEvent: cdk.Duration.seconds(60), }], }); // THEN Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { - Name: 'CustomScheduleTrigger', Type: 'SCHEDULED', - Schedule: 'cron(0 1 1 JAN ? *)', + Schedule: 'cron(0 1 1 JAN ? ?)', Actions: [{ - JobName: { 'Fn::GetAtt': ['myJobC2A9F6D7', 'Name'] }, - SecurityConfiguration: { 'Fn::GetAtt': ['mySecurityConfigurationC2A9F6D7', 'Name'] }, + JobName: { Ref: 'myJob9A6589B3' }, + SecurityConfiguration: { Ref: 'mySecurityConfiguration58B0C573' }, Arguments: { foo: 'bar', key: 'value', }, - Timeout: 300, + Timeout: 5, NotificationProperty: { - NotifyDelayAfter: 5, + NotifyDelayAfter: 1, }, }], - WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + WorkflowName: { Ref: 'myWorkflow931F3265' }, }); }); @@ -254,31 +262,30 @@ describe('workflow with triggers', () => { key: 'value', }, securityConfiguration: securityConfiguration, - delayCloudwatchEvent: cdk.Duration.seconds(5), + delayCloudwatchEvent: cdk.Duration.seconds(60), timeout: cdk.Duration.seconds(5 * 60), }], }); // THEN Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { - Name: 'OnDemandTrigger', Type: 'CONDITIONAL', Actions: [{ - JobName: { 'Fn::GetAtt': ['myJobC2A9F6D7', 'Name'] }, + JobName: { Ref: 'myJob9A6589B3' }, Arguments: { foo: 'bar', key: 'value', }, NotificationProperty: { - NotifyDelayAfter: 0, - NotifyDelayAfterEvent: 'ON_FAILURE', + NotifyDelayAfter: 1, }, + Timeout: 5, }], EventBatchingCondition: { BatchSize: 50, BatchWindow: 300, }, - WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + WorkflowName: { Ref: 'myWorkflow931F3265' }, }); }); @@ -293,7 +300,7 @@ describe('workflow with triggers', () => { }, timeout: cdk.Duration.seconds(5 * 60), securityConfiguration: securityConfiguration, - delayCloudwatchEvent: cdk.Duration.seconds(5), + delayCloudwatchEvent: cdk.Duration.seconds(60), }], predicateCondition: glue.TriggerPredicateCondition.AND, jobPredicates: [{ @@ -308,25 +315,34 @@ describe('workflow with triggers', () => { // THEN Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { - Name: 'ConditionalTrigger', Type: 'CONDITIONAL', Actions: [{ - CrawlerName: { 'Fn::GetAtt': ['myCrawlerC2A9F6D7', 'Name'] }, + CrawlerName: { Ref: 'myCrawler' }, Arguments: { foo: 'bar', key: 'value', }, - Timeout: 300, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, }], Predicate: { - Conditions: ['AND'], - Logical: { - JobName: { 'Fn::GetAtt': ['myJobC2A9F6D7', 'Name'] }, - State: 'SUCCEEDED', - }, - State: 'SUCCEEDED', + Conditions: [ + { + JobName: { Ref: 'myJob9A6589B3' }, + LogicalOperator: 'EQUALS', + State: 'SUCCEEDED', + }, + { + CrawlerName: { Ref: 'myCrawler' }, + LogicalOperator: 'EQUALS', + CrawlState: 'SUCCEEDED', + }, + ], + Logical: 'ALL', }, - WorkflowName: { Ref: 'myWorkflowC2A9F6D7' }, + WorkflowName: { Ref: 'myWorkflow931F3265' }, }); }); @@ -359,7 +375,7 @@ test('workflow with trigger that has both job and crawler as one action', () => executable: glue.JobExecutable.pythonEtl({ glueVersion: glue.GlueVersion.V2_0, pythonVersion: glue.PythonVersion.THREE, - script: glue.Code.fromAsset('myScript'), + script: glue.Code.fromBucket(Bucket.fromBucketName(stack, 'myBucket', 'my-bucket'), 'myKey'), }), }); const crawler = new glueCfn.CfnCrawler(stack, 'myCrawler', { From 5e6cb11595a51f532480d8e9d4caa2bcef3c375f Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Fri, 2 Aug 2024 23:07:30 +0000 Subject: [PATCH 07/18] adding glue README and Integtest file --- packages/@aws-cdk/aws-glue-alpha/README.md | 235 ++++++++++++++++++ .../aws-glue-alpha/rosetta/default.ts-fixture | 3 + .../aws-glue-alpha/test/integ.workflow.ts | 171 +++++++++++++ 3 files changed, 409 insertions(+) create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index 051044a74c8ff..ad623845cedd1 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -594,3 +594,238 @@ new glue.DataQualityRuleset(this, 'MyDataQualityRuleset', { ``` For more information, see [AWS Glue Data Quality](https://docs.aws.amazon.com/glue/latest/dg/glue-data-quality.html). + +## Workflow + +A `Workflow` is a collection of multiple Glue jobs and crawlers that are executed in a DAG. For example, to create a workflow: + +```ts +new glue.Workflow(this, 'MyWorkflow', { + workflowName: 'my_workflow', + description: 'description', + maxConcurrentRuns: 5, + defaultRunProperties: { + key1: 'value1', + key2: 'value2', + }, +}); +``` + +### Add Triggers + +A glue `Workflow` requires triggers to be appeneded to the DAG. These triggers are executed at different stages of the DAG, depending on the type of trigger enabled. + +#### On Demand Triggers + +On Demand triggers are executed manually by the user, or by a third-party service. For example, to add an On Demand trigger to a workflow: + +```ts +declare const myWorkflow: glue.Workflow; +declare const myJob: glue.IJob; +declare const myCrawler: glueCfn.CfnCrawler; +declare const securityConfiguration: glue.ISecurityConfiguration; + +myWorkflow.addOnDemandTrigger(this, 'OnDemandTrigger', { + triggerName: 'on_demand_trigger', + description: 'description', + enabled: true, + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); +``` + +#### Schedule Triggers + +Schedule triggers are executed at a specified time or interval. For example, to add a Schedule trigger to a workflow: + +```ts +declare const myWorkflow: glue.Workflow; +declare const myJob: glue.IJob; +declare const myCrawler: glueCfn.CfnCrawler; +declare const securityConfiguration: glue.ISecurityConfiguration; +declare const schedule: events.Schedule; + +myWorkflow.addCustomScheduleTrigger(this, 'ScheduleTrigger', { + triggerName: 'schedule_trigger', + description: 'description', + enabled: true, + schedule, + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); +``` + +Convinience methods are available to add triggers to a workflow, for daily, weekly, and monthly schedules: + +```ts +declare const myWorkflow: glue.Workflow; +declare const myJob: glue.IJob; +declare const myCrawler: glueCfn.CfnCrawler; +declare const securityConfiguration: glue.ISecurityConfiguration; + +myWorkflow.addDailyScheduleTrigger(this, 'DailyScheduleTrigger', { + triggerName: 'daily_schedule_trigger', + description: 'description', + enabled: true, + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); + +myWorkflow.addWeeklyScheduleTrigger(this, 'WeeklyScheduleTrigger', { + triggerName: 'weekly_schedule_trigger', + description: 'description', + enabled: true, + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); + +myWorkflow.addMonthlyScheduleTrigger(this, 'MonthlyScheduleTrigger', { + triggerName: 'monthly_schedule_trigger', + description: 'description', + enabled: true, + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); +``` + +#### Event Triggers + +Event triggers are executed after a number of events have reached. For example, to add an Event trigger to a workflow: + +```ts +declare const myWorkflow: glue.Workflow; +declare const myJob: glue.IJob; +declare const myCrawler: glueCfn.CfnCrawler; +declare const securityConfiguration: glue.ISecurityConfiguration; + +myWorkflow.addNotifyEventTrigger(this, 'EventTrigger', { + triggerName: 'event_trigger', + description: 'description', + enabled: true, + batchSize: 10, + batchWindow: cdk.Duration.minutes(5), + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); +``` + +#### Conditional Triggers + +Conditional triggers are executed based on a condition. For example, to add a Conditional trigger to a workflow: + +```ts +declare const myWorkflow: glue.Workflow; +declare const myJob: glue.IJob; +declare const myCrawler: glueCfn.CfnCrawler; +declare const predicateJob: glue.IJob; +declare const predicateCrawler: glueCfn.CfnCrawler; +declare const securityConfiguration: glue.ISecurityConfiguration; + +myWorkflow.addConditionalTrigger(this, 'ConditionalTrigger', { + triggerName: 'conditional_trigger', + description: 'description', + enabled: true, + predicateCondition: glue.TriggerPredicateCondition.AND, + jobPredicates: [{ + job: predicateJob, + state: glue.PredicateState.SUCCEEDED, + }], + crawlerPredicates: [{ + crawler: predicateCrawler, + state: glue.PredicateState.SUCCEEDED, + }], + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); +``` diff --git a/packages/@aws-cdk/aws-glue-alpha/rosetta/default.ts-fixture b/packages/@aws-cdk/aws-glue-alpha/rosetta/default.ts-fixture index 047a3eee7b647..e8e34dcaf9a84 100644 --- a/packages/@aws-cdk/aws-glue-alpha/rosetta/default.ts-fixture +++ b/packages/@aws-cdk/aws-glue-alpha/rosetta/default.ts-fixture @@ -7,6 +7,9 @@ import * as s3 from 'aws-cdk-lib/aws-s3'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; import * as kms from 'aws-cdk-lib/aws-kms'; import * as rds from 'aws-cdk-lib/aws-rds'; +import * as cdk from 'aws-cdk-lib'; +import * as glueCfn from 'aws-cdk-lib/aws-glue'; +import * as events from 'aws-cdk-lib/aws-events'; class Fixture extends Stack { constructor(scope: Construct, id: string) { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts new file mode 100644 index 0000000000000..846970c5567d5 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts @@ -0,0 +1,171 @@ +import * as cdk from 'aws-cdk-lib'; +import * as glueCfn from 'aws-cdk-lib/aws-glue'; +import * as path from 'path'; +import * as glue from '../lib'; + +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-cdk-glue-workflow'); + +new glue.Workflow(stack, 'MyWorkflowTask'); +const script = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_world.py')); +const glueJob = new glue.Job(stack, 'MyJob', { + executable: glue.JobExecutable.pythonShell({ + glueVersion: glue.GlueVersion.V1_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), +}); +const crawler = new glueCfn.CfnCrawler(stack, 'MyCrawler', { + databaseName: 'my_database', + targets: { + s3Targets: [{ path: 's3://my_bucket' }], + }, + role: 'my_role', +}); +const predicateGlueJob = new glue.Job(stack, 'MyPredicateJob', { + executable: glue.JobExecutable.pythonShell({ + glueVersion: glue.GlueVersion.V1_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), +}); +const predicateCrawler = new glueCfn.CfnCrawler(stack, 'MyPredicateCrawler', { + databaseName: 'my_database', + targets: { + s3Targets: [{ path: 's3://my_bucket' }], + }, + role: 'my_role', +}); +const securityConfiguration = new glue.SecurityConfiguration(stack, 'MySecurityConfiguration', { + s3Encryption: { + mode: glue.S3EncryptionMode.S3_MANAGED, + }, +}); + +const workflow = new glue.Workflow(stack, 'MyWorkflow', { + workflowName: 'my_workflow', + description: 'my_workflow_description', +}); + +workflow.addConditionalTrigger(stack, 'MyConditionalTrigger', { + triggerName: 'my_conditional_trigger', + description: 'my_conditional_trigger_description', + enabled: true, + predicateCondition: glue.TriggerPredicateCondition.AND, + jobPredicates: [{ + job: predicateGlueJob, + state: glue.PredicateState.SUCCEEDED, + }], + crawlerPredicates: [{ + crawler: predicateCrawler, + state: glue.PredicateState.SUCCEEDED, + }], + actions: [ + { + job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +workflow.addOnDemandTrigger(stack, 'MyOnDemandTrigger', { + triggerName: 'my_on_demand_trigger', + description: 'my_on_demand_trigger_description', + enabled: true, + actions: [ + { + job: glueJob, + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +workflow.addDailyScheduleTrigger(stack, 'MyDailySchedule', { + triggerName: 'my_daily_schedule', + description: 'my_daily_schedule_description', + enabled: true, + actions: [ + { + job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +workflow.addWeeklyScheduleTrigger(stack, 'MyWeeklySchedule', { + triggerName: 'my_weekly_schedule', + description: 'my_weekly_schedule_description', + enabled: true, + actions: [ + { + job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +workflow.addMonthlyScheduleTrigger(stack, 'MyMonthlySchedule', { + triggerName: 'my_monthly_schedule', + description: 'my_monthly_schedule_description', + enabled: true, + actions: [ + { + job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); From d9090d70f4bd53f500ed689992436c06664783cb Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Sat, 3 Aug 2024 12:20:13 +0000 Subject: [PATCH 08/18] missing tests from integ --- .../aws-glue-alpha/test/integ.workflow.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts index 846970c5567d5..dee96cf402213 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts @@ -169,3 +169,30 @@ workflow.addMonthlyScheduleTrigger(stack, 'MyMonthlySchedule', { }, ], }); + +workflow.addNotifyEventTrigger('MyNotifyEvent', { + triggerName: 'my_notify_event', + description: 'my_notify_event_description', + enabled: true, + batchSize: 10, + batchWindow: cdk.Duration.minutes(10), + actions: [ + { + job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +glue.Workflow.fromWorkflowName(stack, 'imported-workflow', workflow.workflowName); From 4c5cb58dc425bd737d7b46c21f37cde5416c670b Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Sat, 3 Aug 2024 12:22:26 +0000 Subject: [PATCH 09/18] changing scope to belong to workflow --- .../@aws-cdk/aws-glue-alpha/lib/workflow.ts | 56 +++++-------------- .../aws-glue-alpha/test/integ.workflow.ts | 10 ++-- 2 files changed, 19 insertions(+), 47 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts index 92bcc9a09fef3..0b2bbf0baa064 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts @@ -270,13 +270,9 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * Adds a trigger to the workflow to run on demand. */ - public addOnDemandTrigger( - scope: Construct, - id: string, - props: TriggerProps, - ): void { + public addOnDemandTrigger(id: string, props: TriggerProps): void { this.triggers.push( - new CfnTrigger(scope, id, { + new CfnTrigger(this, id, { name: props.triggerName, type: 'ON_DEMAND', workflowName: this.workflowName, @@ -289,13 +285,9 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * Adds a trigger to the workflow to run on a daily schedule. */ - public addDailyScheduleTrigger( - scope: Construct, - id: string, - props: TriggerProps, - ): void { + public addDailyScheduleTrigger(id: string, props: TriggerProps): void { this.triggers.push( - new CfnTrigger(scope, id, { + new CfnTrigger(this, id, { name: props.triggerName, type: 'SCHEDULED', schedule: 'cron(0 1 * * ? *)', @@ -309,13 +301,9 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * Adds a trigger to the workflow to run on a weekly schedule. */ - public addWeeklyScheduleTrigger( - scope: Construct, - id: string, - props: TriggerProps, - ): void { + public addWeeklyScheduleTrigger(id: string, props: TriggerProps): void { this.triggers.push( - new CfnTrigger(scope, id, { + new CfnTrigger(this, id, { name: props.triggerName, type: 'SCHEDULED', schedule: 'cron(0 1 ? * MON *)', @@ -329,13 +317,9 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * Adds a trigger to the workflow to run on a monthly schedule. */ - public addMonthlyScheduleTrigger( - scope: Construct, - id: string, - props: TriggerProps, - ): void { + public addMonthlyScheduleTrigger(id: string, props: TriggerProps): void { this.triggers.push( - new CfnTrigger(scope, id, { + new CfnTrigger(this, id, { name: props.triggerName, type: 'SCHEDULED', schedule: 'cron(0 1 1 * ? *)', @@ -349,13 +333,9 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * Adds a trigger to the workflow to run on a custom schedule. */ - public addCustomScheduleTrigger( - scope: Construct, - id: string, - props: ScheduleTriggerProps, - ): void { + public addCustomScheduleTrigger(id: string, props: ScheduleTriggerProps): void { this.triggers.push( - new CfnTrigger(scope, id, { + new CfnTrigger(this, id, { name: props.triggerName, type: 'SCHEDULED', schedule: props.schedule.expressionString, @@ -369,13 +349,9 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * Adds a trigger to the workflow to run on a notification event. */ - public addNotifyEventTrigger( - scope: Construct, - id: string, - props: NotificationTriggerProps, - ): void { + public addNotifyEventTrigger(id: string, props: NotificationTriggerProps): void { this.triggers.push( - new CfnTrigger(scope, id, { + new CfnTrigger(this, id, { name: props.triggerName, type: 'CONDITIONAL', workflowName: this.workflowName, @@ -392,11 +368,7 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * Adds a trigger to the workflow to run on a conditional event. */ - public addConditionalTrigger( - scope: Construct, - id: string, - props: ConditionalTriggerProps, - ): void { + public addConditionalTrigger(id: string, props: ConditionalTriggerProps): void { this.validatePredicates(props.jobPredicates, props.crawlerPredicates); const conditions: CfnTrigger.ConditionProperty[] = []; @@ -422,7 +394,7 @@ abstract class WorkflowBase extends Resource implements IWorkflow { } this.triggers.push( - new CfnTrigger(scope, id, { + new CfnTrigger(this, id, { name: props.triggerName, type: 'CONDITIONAL', workflowName: this.workflowName, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts index dee96cf402213..cc6c9f91c768c 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts @@ -48,7 +48,7 @@ const workflow = new glue.Workflow(stack, 'MyWorkflow', { description: 'my_workflow_description', }); -workflow.addConditionalTrigger(stack, 'MyConditionalTrigger', { +workflow.addConditionalTrigger('MyConditionalTrigger', { triggerName: 'my_conditional_trigger', description: 'my_conditional_trigger_description', enabled: true, @@ -80,7 +80,7 @@ workflow.addConditionalTrigger(stack, 'MyConditionalTrigger', { ], }); -workflow.addOnDemandTrigger(stack, 'MyOnDemandTrigger', { +workflow.addOnDemandTrigger('MyOnDemandTrigger', { triggerName: 'my_on_demand_trigger', description: 'my_on_demand_trigger_description', enabled: true, @@ -101,7 +101,7 @@ workflow.addOnDemandTrigger(stack, 'MyOnDemandTrigger', { ], }); -workflow.addDailyScheduleTrigger(stack, 'MyDailySchedule', { +workflow.addDailyScheduleTrigger('MyDailySchedule', { triggerName: 'my_daily_schedule', description: 'my_daily_schedule_description', enabled: true, @@ -124,7 +124,7 @@ workflow.addDailyScheduleTrigger(stack, 'MyDailySchedule', { ], }); -workflow.addWeeklyScheduleTrigger(stack, 'MyWeeklySchedule', { +workflow.addWeeklyScheduleTrigger('MyWeeklySchedule', { triggerName: 'my_weekly_schedule', description: 'my_weekly_schedule_description', enabled: true, @@ -147,7 +147,7 @@ workflow.addWeeklyScheduleTrigger(stack, 'MyWeeklySchedule', { ], }); -workflow.addMonthlyScheduleTrigger(stack, 'MyMonthlySchedule', { +workflow.addMonthlyScheduleTrigger('MyMonthlySchedule', { triggerName: 'my_monthly_schedule', description: 'my_monthly_schedule_description', enabled: true, From 9d0e4792942a53317c30e260d7715035effde97b Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Sat, 3 Aug 2024 12:22:58 +0000 Subject: [PATCH 10/18] incorrect implementation of interface, imports weren't able to access methods --- .../@aws-cdk/aws-glue-alpha/lib/workflow.ts | 69 ++++++++++++++++++- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts index 0b2bbf0baa064..2433d7a13d643 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts @@ -226,6 +226,62 @@ export interface IWorkflow extends IResource { * @attribute */ readonly workflowName: string; + + /** + * Adds a trigger to the workflow to run on demand. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addOnDemandTrigger(id: string, props: TriggerProps): void; + + /** + * Adds a trigger to the workflow to run on a daily schedule. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addDailyScheduleTrigger(id: string, props: TriggerProps): void; + + /** + * Adds a trigger to the workflow to run on a weekly schedule. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addWeeklyScheduleTrigger(id: string, props: TriggerProps): void; + + /** + * Adds a trigger to the workflow to run on a monthly schedule. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addMonthlyScheduleTrigger(id: string, props: TriggerProps): void; + + /** + * Adds a trigger to the workflow to run on a custom schedule. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addCustomScheduleTrigger(id: string, props: ScheduleTriggerProps): void; + + /** + * Adds a trigger to the workflow to run on a notification event. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addNotifyEventTrigger(id: string, props: NotificationTriggerProps): void; + + /** + * Adds a trigger to the workflow to run on a conditional event. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addConditionalTrigger(id: string, props: ConditionalTriggerProps): void; } /** @@ -262,7 +318,14 @@ export interface WorkflowProps { } abstract class WorkflowBase extends Resource implements IWorkflow { + /** + * The ARN of the workflow. + */ public abstract readonly workflowArn: string; + + /** + * The name of the workflow. + */ public abstract readonly workflowName: string; private readonly triggers: CfnTrigger[] = []; @@ -451,7 +514,7 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * A Glue workflow. */ -export class Workflow extends WorkflowBase implements IWorkflow { +export class Workflow extends WorkflowBase { /** * Import an existing workflow, using its ARN. */ @@ -460,7 +523,7 @@ export class Workflow extends WorkflowBase implements IWorkflow { id: string, workflowArn: string, ): IWorkflow { - class Import extends WorkflowBase implements IWorkflow { + class Import extends WorkflowBase { public readonly workflowArn = workflowArn; public readonly workflowName = cdk.Arn.extractResourceName( workflowArn, @@ -479,7 +542,7 @@ export class Workflow extends WorkflowBase implements IWorkflow { id: string, workflowName: string, ): IWorkflow { - class Import extends WorkflowBase implements IWorkflow { + class Import extends WorkflowBase { public readonly workflowArn = Workflow.buildWorkflowArn( scope, workflowName, From e0442afd628538da75a96a8fd044b82522983c87 Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Sat, 3 Aug 2024 13:52:16 +0000 Subject: [PATCH 11/18] removing stack from test --- .../aws-glue-alpha/test/workflow.test.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts index 6d261c173e906..300dee010ae49 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts @@ -76,7 +76,7 @@ describe('workflow with triggers', () => { test('onDemand', () => { // WHEN - workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { + workflow.addOnDemandTrigger('OnDemandTrigger', { triggerName: 'OnDemandTrigger', actions: [{ job: job, @@ -112,7 +112,7 @@ describe('workflow with triggers', () => { test('dailySchedule', () => { // WHEN - workflow.addDailyScheduleTrigger(stack, 'DailyScheduleTrigger', { + workflow.addDailyScheduleTrigger('DailyScheduleTrigger', { actions: [{ crawler: crawler, arguments: { @@ -147,7 +147,7 @@ describe('workflow with triggers', () => { test('weeklySchedule', () => { // WHEN - workflow.addWeeklyScheduleTrigger(stack, 'WeeklyScheduleTrigger', { + workflow.addWeeklyScheduleTrigger('WeeklyScheduleTrigger', { actions: [{ job: job, arguments: { @@ -182,7 +182,7 @@ describe('workflow with triggers', () => { test('monthlySchedule', () => { // WHEN - workflow.addMonthlyScheduleTrigger(stack, 'MonthlyScheduleTrigger', { + workflow.addMonthlyScheduleTrigger('MonthlyScheduleTrigger', { actions: [{ crawler: crawler, arguments: { @@ -216,7 +216,7 @@ describe('workflow with triggers', () => { test('customSchedule', () => { // WHEN - workflow.addCustomScheduleTrigger(stack, 'CustomScheduleTrigger', { + workflow.addCustomScheduleTrigger('CustomScheduleTrigger', { schedule: Schedule.cron({ minute: '0', hour: '1', day: '1', month: 'JAN', year: '?' }), actions: [{ job: job, @@ -252,7 +252,7 @@ describe('workflow with triggers', () => { test('notifyEvent', () => { // WHEN - workflow.addNotifyEventTrigger(stack, 'OnDemandTrigger', { + workflow.addNotifyEventTrigger('OnDemandTrigger', { batchSize: 50, batchWindow: cdk.Duration.minutes(5), actions: [{ @@ -291,7 +291,7 @@ describe('workflow with triggers', () => { test('conditional', () => { // WHEN - workflow.addConditionalTrigger(stack, 'ConditionalTrigger', { + workflow.addConditionalTrigger('ConditionalTrigger', { actions: [{ crawler: crawler, arguments: { @@ -349,7 +349,7 @@ describe('workflow with triggers', () => { test('conditional with no predicates', () => { // WHEN expect(() => - workflow.addConditionalTrigger(stack, 'ConditionalTrigger', { + workflow.addConditionalTrigger('ConditionalTrigger', { actions: [{ crawler: crawler, arguments: { @@ -387,7 +387,7 @@ test('workflow with trigger that has both job and crawler as one action', () => }); // WHEN - expect(() => workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { + expect(() => workflow.addOnDemandTrigger('OnDemandTrigger', { actions: [{ job: job, crawler: crawler, @@ -404,7 +404,7 @@ test('workflow with trigger that has neither job nor crawler as one action', () const workflow = new glue.Workflow(stack, 'myWorkflow'); // WHEN - expect(() => workflow.addOnDemandTrigger(stack, 'OnDemandTrigger', { + expect(() => workflow.addOnDemandTrigger('OnDemandTrigger', { actions: [{ arguments: { foo: 'bar', From da57f12bc1bbe1fda1370dc682a10f3eb8e48b5e Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Sat, 3 Aug 2024 13:53:02 +0000 Subject: [PATCH 12/18] adding tests for imports --- .../aws-glue-alpha/test/integ.workflow.ts | 4 +- .../aws-glue-alpha/test/workflow.test.ts | 113 ++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts index cc6c9f91c768c..8ebebd59c2359 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts @@ -195,4 +195,6 @@ workflow.addNotifyEventTrigger('MyNotifyEvent', { ], }); -glue.Workflow.fromWorkflowName(stack, 'imported-workflow', workflow.workflowName); +const importedWorkflowByName = glue.Workflow.fromWorkflowName(stack, 'imported-workflow-name', workflow.workflowName); + +glue.Workflow.fromWorkflowArn(stack, 'imported-workflow-arn', importedWorkflowByName.workflowArn); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts index 300dee010ae49..0a38b9676833e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts @@ -43,6 +43,119 @@ test('workflow with props', () => { }); }); +describe('imported workflow', () => { + let stack: cdk.Stack; + let job: glue.Job; + let securityConfiguration: glue.SecurityConfiguration; + let triggerProps: glue.TriggerProps; + + beforeEach(() => { + stack = new cdk.Stack(); + job = new glue.Job(stack, 'myJob', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script: glue.Code.fromBucket(Bucket.fromBucketName(stack, 'myBucket', 'my-bucket'), 'myKey'), + }), + }); + securityConfiguration = new glue.SecurityConfiguration(stack, 'mySecurityConfiguration', { + s3Encryption: { + mode: glue.S3EncryptionMode.S3_MANAGED, + }, + }); + triggerProps = { + actions: [{ + job: job, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(60), + }], + }; + }); + + test('fromWorkflowArn', () => { + // GIVEN + const workflowArn = 'arn:aws:glue:us-east-1:123456789012:workflow/myWorkflow'; + const predicateJob = new glue.Job(stack, 'myJob2', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script: glue.Code.fromBucket(Bucket.fromBucketName(stack, 'myPredicateBucket', 'my-bucket'), 'myKey'), + }), + }); + + // WHEN + const imported = glue.Workflow.fromWorkflowArn(stack, 'myWorkflow', workflowArn); + imported.addConditionalTrigger('ConditionalTrigger', { + ...triggerProps, + predicateCondition: glue.TriggerPredicateCondition.AND, + jobPredicates: [{ + job: predicateJob, + state: glue.PredicateState.SUCCEEDED, + }], + }); + + // THEN + expect(imported.workflowArn).toEqual(workflowArn); + expect(imported.workflowName).toEqual('myWorkflow'); + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Type: 'CONDITIONAL', + Actions: [{ + JobName: { Ref: 'myJob9A6589B3' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + }], + Predicate: { + Conditions: [ + { + JobName: { Ref: 'myJob2902BF595' }, + LogicalOperator: 'EQUALS', + State: 'SUCCEEDED', + }, + ], + Logical: 'ALL', + }, + WorkflowName: 'myWorkflow', + }); + }); + + test('fromWorkflowName', () => { + // WHEN + const imported = glue.Workflow.fromWorkflowName(stack, 'myWorkflow', 'myWorkflow'); + imported.addOnDemandTrigger('OnDemandTrigger', triggerProps); + + // THEN + expect(imported.workflowName).toEqual('myWorkflow'); + expect(imported.workflowArn).toMatch(/^arn:.+:glue:.+:.+:workflow\/myWorkflow$/); + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Type: 'ON_DEMAND', + Actions: [{ + JobName: { Ref: 'myJob9A6589B3' }, + SecurityConfiguration: { Ref: 'mySecurityConfiguration58B0C573' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + }], + WorkflowName: 'myWorkflow', + }); + }); +}); + describe('workflow with triggers', () => { let stack: cdk.Stack; let workflow: glue.Workflow; From 1c9ca93b87485a7b4e28649fc391a77c96584589 Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Mon, 5 Aug 2024 10:15:49 +0000 Subject: [PATCH 13/18] reordering trigger --- .../aws-glue-alpha/test/integ.workflow.ts | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts index 8ebebd59c2359..69506e643f676 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts @@ -2,12 +2,17 @@ import * as cdk from 'aws-cdk-lib'; import * as glueCfn from 'aws-cdk-lib/aws-glue'; import * as path from 'path'; import * as glue from '../lib'; +import * as integ from '@aws-cdk/integ-tests-alpha'; +import * as iam from 'aws-cdk-lib/aws-iam'; const app = new cdk.App(); const stack = new cdk.Stack(app, 'aws-cdk-glue-workflow'); new glue.Workflow(stack, 'MyWorkflowTask'); +const glueRole = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('glue.amazonaws.com'), +}); const script = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_world.py')); const glueJob = new glue.Job(stack, 'MyJob', { executable: glue.JobExecutable.pythonShell({ @@ -21,7 +26,7 @@ const crawler = new glueCfn.CfnCrawler(stack, 'MyCrawler', { targets: { s3Targets: [{ path: 's3://my_bucket' }], }, - role: 'my_role', + role: glueRole.roleArn, }); const predicateGlueJob = new glue.Job(stack, 'MyPredicateJob', { executable: glue.JobExecutable.pythonShell({ @@ -35,7 +40,7 @@ const predicateCrawler = new glueCfn.CfnCrawler(stack, 'MyPredicateCrawler', { targets: { s3Targets: [{ path: 's3://my_bucket' }], }, - role: 'my_role', + role: glueRole.roleArn, }); const securityConfiguration = new glue.SecurityConfiguration(stack, 'MySecurityConfiguration', { s3Encryption: { @@ -48,23 +53,13 @@ const workflow = new glue.Workflow(stack, 'MyWorkflow', { description: 'my_workflow_description', }); -workflow.addConditionalTrigger('MyConditionalTrigger', { - triggerName: 'my_conditional_trigger', - description: 'my_conditional_trigger_description', +workflow.addOnDemandTrigger('MyOnDemandTrigger', { + triggerName: 'my_on_demand_trigger', + description: 'my_on_demand_trigger_description', enabled: true, - predicateCondition: glue.TriggerPredicateCondition.AND, - jobPredicates: [{ - job: predicateGlueJob, - state: glue.PredicateState.SUCCEEDED, - }], - crawlerPredicates: [{ - crawler: predicateCrawler, - state: glue.PredicateState.SUCCEEDED, - }], actions: [ { job: glueJob, - delayCloudwatchEvent: cdk.Duration.minutes(5), arguments: { '--arg1': 'value1', }, @@ -73,20 +68,29 @@ workflow.addConditionalTrigger('MyConditionalTrigger', { }, { crawler: crawler, - delayCloudwatchEvent: cdk.Duration.minutes(5), securityConfiguration: securityConfiguration, timeout: cdk.Duration.minutes(10), }, ], }); -workflow.addOnDemandTrigger('MyOnDemandTrigger', { - triggerName: 'my_on_demand_trigger', - description: 'my_on_demand_trigger_description', +workflow.addConditionalTrigger('MyConditionalTrigger', { + triggerName: 'my_conditional_trigger', + description: 'my_conditional_trigger_description', enabled: true, + predicateCondition: glue.TriggerPredicateCondition.AND, + jobPredicates: [{ + job: predicateGlueJob, + state: glue.PredicateState.SUCCEEDED, + }], + crawlerPredicates: [{ + crawler: predicateCrawler, + state: glue.PredicateState.SUCCEEDED, + }], actions: [ { job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), arguments: { '--arg1': 'value1', }, @@ -95,6 +99,7 @@ workflow.addOnDemandTrigger('MyOnDemandTrigger', { }, { crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), securityConfiguration: securityConfiguration, timeout: cdk.Duration.minutes(10), }, @@ -197,4 +202,10 @@ workflow.addNotifyEventTrigger('MyNotifyEvent', { const importedWorkflowByName = glue.Workflow.fromWorkflowName(stack, 'imported-workflow-name', workflow.workflowName); -glue.Workflow.fromWorkflowArn(stack, 'imported-workflow-arn', importedWorkflowByName.workflowArn); \ No newline at end of file +glue.Workflow.fromWorkflowArn(stack, 'imported-workflow-arn', importedWorkflowByName.workflowArn); + +new integ.IntegTest(app, 'aws-cdk-glue-table-integ', { + testCases: [stack], +}); + +app.synth(); From 9b89872e0daabc54835b537fc60f49b014aff7d7 Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Mon, 5 Aug 2024 11:05:16 +0000 Subject: [PATCH 14/18] wrong type, and wrong logical operator --- .../@aws-cdk/aws-glue-alpha/lib/workflow.ts | 39 ++++++++++--------- .../aws-glue-alpha/test/workflow.test.ts | 6 +-- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts index 2433d7a13d643..1094a85d08a95 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts @@ -13,7 +13,7 @@ export enum TriggerPredicateCondition { /** * Defines when all predicates are met. */ - AND = 'ALL', + AND = 'AND', /** * Defines when any predicate is met. @@ -142,22 +142,27 @@ export interface TriggerProps { readonly description?: string; /** - * Whether this trigger is enabled. - * - * @default true + * The actions to be executed when this trigger fires. */ - readonly enabled?: boolean; + readonly actions: Action[]; +} +/** + * Properties for a trigger that can be enabled or disabled. + */ +export interface ActivatableTriggerProps extends TriggerProps { /** - * The actions to be executed when this trigger fires. + * Whether the trigger is enabled. + * + * @default true */ - readonly actions: Action[]; + readonly enabled?: boolean; } /** * Properties for defining a new schedule trigger, to be assigned to a workflow. */ -export interface ScheduleTriggerProps extends TriggerProps { +export interface ScheduleTriggerProps extends ActivatableTriggerProps { /** * The schedule for this trigger. */ @@ -186,7 +191,7 @@ export interface NotificationTriggerProps extends TriggerProps { /** * Properties for defining a new conditional trigger, to be assigned to a workflow. */ -export interface ConditionalTriggerProps extends TriggerProps { +export interface ConditionalTriggerProps extends ActivatableTriggerProps { /** * The condition to be evaluated. * @@ -241,7 +246,7 @@ export interface IWorkflow extends IResource { * @param id The identifier for the trigger. * @param props The properties for the trigger. */ - addDailyScheduleTrigger(id: string, props: TriggerProps): void; + addDailyScheduleTrigger(id: string, props: ActivatableTriggerProps): void; /** * Adds a trigger to the workflow to run on a weekly schedule. @@ -249,7 +254,7 @@ export interface IWorkflow extends IResource { * @param id The identifier for the trigger. * @param props The properties for the trigger. */ - addWeeklyScheduleTrigger(id: string, props: TriggerProps): void; + addWeeklyScheduleTrigger(id: string, props: ActivatableTriggerProps): void; /** * Adds a trigger to the workflow to run on a monthly schedule. @@ -257,7 +262,7 @@ export interface IWorkflow extends IResource { * @param id The identifier for the trigger. * @param props The properties for the trigger. */ - addMonthlyScheduleTrigger(id: string, props: TriggerProps): void; + addMonthlyScheduleTrigger(id: string, props: ActivatableTriggerProps): void; /** * Adds a trigger to the workflow to run on a custom schedule. @@ -339,7 +344,6 @@ abstract class WorkflowBase extends Resource implements IWorkflow { name: props.triggerName, type: 'ON_DEMAND', workflowName: this.workflowName, - startOnCreation: props.enabled ?? true, actions: props.actions?.map(action => this.renderAction(action)), }), ); @@ -348,7 +352,7 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * Adds a trigger to the workflow to run on a daily schedule. */ - public addDailyScheduleTrigger(id: string, props: TriggerProps): void { + public addDailyScheduleTrigger(id: string, props: ActivatableTriggerProps): void { this.triggers.push( new CfnTrigger(this, id, { name: props.triggerName, @@ -364,7 +368,7 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * Adds a trigger to the workflow to run on a weekly schedule. */ - public addWeeklyScheduleTrigger(id: string, props: TriggerProps): void { + public addWeeklyScheduleTrigger(id: string, props: ActivatableTriggerProps): void { this.triggers.push( new CfnTrigger(this, id, { name: props.triggerName, @@ -380,7 +384,7 @@ abstract class WorkflowBase extends Resource implements IWorkflow { /** * Adds a trigger to the workflow to run on a monthly schedule. */ - public addMonthlyScheduleTrigger(id: string, props: TriggerProps): void { + public addMonthlyScheduleTrigger(id: string, props: ActivatableTriggerProps): void { this.triggers.push( new CfnTrigger(this, id, { name: props.triggerName, @@ -416,9 +420,8 @@ abstract class WorkflowBase extends Resource implements IWorkflow { this.triggers.push( new CfnTrigger(this, id, { name: props.triggerName, - type: 'CONDITIONAL', + type: 'EVENT', workflowName: this.workflowName, - startOnCreation: props.enabled ?? true, actions: props.actions?.map(action => this.renderAction(action)) ?? [], eventBatchingCondition: { batchSize: props.batchSize ?? 1, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts index 0a38b9676833e..e7a0e130a18b4 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts @@ -123,7 +123,7 @@ describe('imported workflow', () => { State: 'SUCCEEDED', }, ], - Logical: 'ALL', + Logical: 'AND', }, WorkflowName: 'myWorkflow', }); @@ -382,7 +382,7 @@ describe('workflow with triggers', () => { // THEN Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { - Type: 'CONDITIONAL', + Type: 'EVENT', Actions: [{ JobName: { Ref: 'myJob9A6589B3' }, Arguments: { @@ -453,7 +453,7 @@ describe('workflow with triggers', () => { CrawlState: 'SUCCEEDED', }, ], - Logical: 'ALL', + Logical: 'AND', }, WorkflowName: { Ref: 'myWorkflow931F3265' }, }); From 5aefde6a22875e0e7c324359e42867bbe3278b73 Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Mon, 5 Aug 2024 11:05:30 +0000 Subject: [PATCH 15/18] integ test --- ...9be7858a12b228a2ae6e5c10faccd9097b1e855.py | 1 + .../aws-cdk-glue-workflow.assets.json | 32 + .../aws-cdk-glue-workflow.template.json | 642 ++++++++++ ...efaultTestDeployAssert8BFB5B70.assets.json | 19 + ...aultTestDeployAssert8BFB5B70.template.json | 36 + .../test/integ.workflow.js.snapshot/cdk.out | 1 + .../integ.workflow.js.snapshot/integ.json | 12 + .../integ.workflow.js.snapshot/manifest.json | 239 ++++ .../test/integ.workflow.js.snapshot/tree.json | 1090 +++++++++++++++++ .../aws-glue-alpha/test/integ.workflow.ts | 42 +- 10 files changed, 2102 insertions(+), 12 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.assets.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.template.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/cdk.out create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/integ.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/manifest.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/tree.json diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py new file mode 100644 index 0000000000000..e75154b7c390f --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py @@ -0,0 +1 @@ +print("hello world") \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.assets.json new file mode 100644 index 0000000000000..0bd473a04b737 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.assets.json @@ -0,0 +1,32 @@ +{ + "version": "36.0.0", + "files": { + "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { + "source": { + "path": "asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "085a2e2b10fd8c98d1a4d022657be795839a4fe0286e2c41082219f45aecbbec": { + "source": { + "path": "aws-cdk-glue-workflow.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "085a2e2b10fd8c98d1a4d022657be795839a4fe0286e2c41082219f45aecbbec.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.template.json new file mode 100644 index 0000000000000..498850985e37d --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.template.json @@ -0,0 +1,642 @@ +{ + "Resources": { + "MyWorkflowTask5BAF3B32": { + "Type": "AWS::Glue::Workflow" + }, + "Role1ABCC5F0": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "MyJobServiceRoleE6853162": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "MyJobServiceRoleDefaultPolicy3BDBBD28": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyJobServiceRoleDefaultPolicy3BDBBD28", + "Roles": [ + { + "Ref": "MyJobServiceRoleE6853162" + } + ] + } + }, + "MyJob8719E923": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "pythonshell", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "DefaultArguments": { + "--job-language": "python" + }, + "GlueVersion": "1.0", + "Role": { + "Fn::GetAtt": [ + "MyJobServiceRoleE6853162", + "Arn" + ] + } + } + }, + "MyCrawler": { + "Type": "AWS::Glue::Crawler", + "Properties": { + "DatabaseName": "my_database", + "Role": { + "Fn::GetAtt": [ + "Role1ABCC5F0", + "Arn" + ] + }, + "Targets": { + "S3Targets": [ + { + "Path": "s3://my_bucket" + } + ] + } + } + }, + "MyPredicateJobServiceRoleE1F838CC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "MyPredicateJobServiceRoleDefaultPolicy87CB55E2": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyPredicateJobServiceRoleDefaultPolicy87CB55E2", + "Roles": [ + { + "Ref": "MyPredicateJobServiceRoleE1F838CC" + } + ] + } + }, + "MyPredicateJobF30A7B45": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "pythonshell", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "DefaultArguments": { + "--job-language": "python" + }, + "GlueVersion": "1.0", + "Role": { + "Fn::GetAtt": [ + "MyPredicateJobServiceRoleE1F838CC", + "Arn" + ] + } + } + }, + "MyPredicateCrawler": { + "Type": "AWS::Glue::Crawler", + "Properties": { + "DatabaseName": "my_database", + "Role": { + "Fn::GetAtt": [ + "Role1ABCC5F0", + "Arn" + ] + }, + "Targets": { + "S3Targets": [ + { + "Path": "s3://my_bucket" + } + ] + } + } + }, + "MySecurityConfiguration4AB9DD39": { + "Type": "AWS::Glue::SecurityConfiguration", + "Properties": { + "EncryptionConfiguration": { + "S3Encryptions": [ + { + "S3EncryptionMode": "SSE-S3" + } + ] + }, + "Name": "awscdkglueworkflowMySecurityConfigurationBBF7B904" + } + }, + "MyWorkflowBA72662B": { + "Type": "AWS::Glue::Workflow", + "Properties": { + "Description": "my_on_demand_workflow_description", + "Name": "my_on_demand_workflow" + } + }, + "MyWorkflowMyOnDemandTrigger2A682A63": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": {}, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": {}, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "Name": "my_on_demand_trigger", + "Type": "ON_DEMAND", + "WorkflowName": { + "Ref": "MyWorkflowBA72662B" + } + } + }, + "MyWorkflowMyConditionalTrigger389C8949": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "Name": "my_conditional_trigger", + "Predicate": { + "Conditions": [ + { + "JobName": { + "Ref": "MyPredicateJobF30A7B45" + }, + "LogicalOperator": "EQUALS", + "State": "SUCCEEDED" + }, + { + "CrawlState": "SUCCEEDED", + "CrawlerName": { + "Ref": "MyPredicateCrawler" + }, + "LogicalOperator": "EQUALS" + } + ], + "Logical": "AND" + }, + "StartOnCreation": true, + "Type": "CONDITIONAL", + "WorkflowName": { + "Ref": "MyWorkflowBA72662B" + } + } + }, + "MyDailyWorkflow3684C8E1": { + "Type": "AWS::Glue::Workflow", + "Properties": { + "Description": "my_daily_workflow_description", + "Name": "my_daily_workflow" + } + }, + "MyDailyWorkflowMyDailySchedule2DF51EB8": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "Name": "my_daily_schedule", + "Schedule": "cron(0 1 * * ? *)", + "StartOnCreation": true, + "Type": "SCHEDULED", + "WorkflowName": { + "Ref": "MyDailyWorkflow3684C8E1" + } + } + }, + "MyWeeklyWorkflowFC4A11C3": { + "Type": "AWS::Glue::Workflow", + "Properties": { + "Description": "my_weekly_workflow_description", + "Name": "my_weekly_workflow" + } + }, + "MyWeeklyWorkflowMyWeeklySchedule66671664": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "Name": "my_weekly_schedule", + "Schedule": "cron(0 1 ? * MON *)", + "StartOnCreation": true, + "Type": "SCHEDULED", + "WorkflowName": { + "Ref": "MyWeeklyWorkflowFC4A11C3" + } + } + }, + "MyMonthlyWorkflow5B4B0907": { + "Type": "AWS::Glue::Workflow", + "Properties": { + "Description": "my_monthly_workflow_description", + "Name": "my_monthly_workflow" + } + }, + "MyMonthlyWorkflowMyMonthlySchedule14F546AC": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "Name": "my_monthly_schedule", + "Schedule": "cron(0 1 1 * ? *)", + "StartOnCreation": true, + "Type": "SCHEDULED", + "WorkflowName": { + "Ref": "MyMonthlyWorkflow5B4B0907" + } + } + }, + "MyEventWorkflow0AA02AF3": { + "Type": "AWS::Glue::Workflow", + "Properties": { + "Description": "my_event_workflow_description", + "Name": "my_event_workflow" + } + }, + "MyEventWorkflowMyNotifyEvent4590A017": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "EventBatchingCondition": { + "BatchSize": 10, + "BatchWindow": 600 + }, + "Name": "my_notify_event", + "Type": "EVENT", + "WorkflowName": { + "Ref": "MyEventWorkflow0AA02AF3" + } + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json new file mode 100644 index 0000000000000..cc6198ec6ee39 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json @@ -0,0 +1,19 @@ +{ + "version": "36.0.0", + "files": { + "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { + "source": { + "path": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json new file mode 100644 index 0000000000000..ad9d0fb73d1dd --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json @@ -0,0 +1,36 @@ +{ + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/cdk.out new file mode 100644 index 0000000000000..1f0068d32659a --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/cdk.out @@ -0,0 +1 @@ +{"version":"36.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/integ.json new file mode 100644 index 0000000000000..94388279220a8 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/integ.json @@ -0,0 +1,12 @@ +{ + "version": "36.0.0", + "testCases": { + "aws-cdk-glue-table-integ/DefaultTest": { + "stacks": [ + "aws-cdk-glue-workflow" + ], + "assertionStack": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert", + "assertionStackName": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/manifest.json new file mode 100644 index 0000000000000..2101fabf1d9ae --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/manifest.json @@ -0,0 +1,239 @@ +{ + "version": "36.0.0", + "artifacts": { + "aws-cdk-glue-workflow.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "aws-cdk-glue-workflow.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "aws-cdk-glue-workflow": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "aws-cdk-glue-workflow.template.json", + "terminationProtection": false, + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/085a2e2b10fd8c98d1a4d022657be795839a4fe0286e2c41082219f45aecbbec.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "aws-cdk-glue-workflow.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "aws-cdk-glue-workflow.assets" + ], + "metadata": { + "/aws-cdk-glue-workflow/MyWorkflowTask/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWorkflowTask5BAF3B32" + } + ], + "/aws-cdk-glue-workflow/Role/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "Role1ABCC5F0" + } + ], + "/aws-cdk-glue-workflow/MyJob/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyJobServiceRoleE6853162" + } + ], + "/aws-cdk-glue-workflow/MyJob/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyJobServiceRoleDefaultPolicy3BDBBD28" + } + ], + "/aws-cdk-glue-workflow/MyJob/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyJob8719E923" + } + ], + "/aws-cdk-glue-workflow/MyCrawler": [ + { + "type": "aws:cdk:logicalId", + "data": "MyCrawler" + } + ], + "/aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPredicateJobServiceRoleE1F838CC" + } + ], + "/aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPredicateJobServiceRoleDefaultPolicy87CB55E2" + } + ], + "/aws-cdk-glue-workflow/MyPredicateJob/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPredicateJobF30A7B45" + } + ], + "/aws-cdk-glue-workflow/MyPredicateCrawler": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPredicateCrawler" + } + ], + "/aws-cdk-glue-workflow/MySecurityConfiguration/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MySecurityConfiguration4AB9DD39" + } + ], + "/aws-cdk-glue-workflow/MyWorkflow/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWorkflowBA72662B" + } + ], + "/aws-cdk-glue-workflow/MyWorkflow/MyOnDemandTrigger": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWorkflowMyOnDemandTrigger2A682A63" + } + ], + "/aws-cdk-glue-workflow/MyWorkflow/MyConditionalTrigger": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWorkflowMyConditionalTrigger389C8949" + } + ], + "/aws-cdk-glue-workflow/MyDailyWorkflow/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyDailyWorkflow3684C8E1" + } + ], + "/aws-cdk-glue-workflow/MyDailyWorkflow/MyDailySchedule": [ + { + "type": "aws:cdk:logicalId", + "data": "MyDailyWorkflowMyDailySchedule2DF51EB8" + } + ], + "/aws-cdk-glue-workflow/MyWeeklyWorkflow/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWeeklyWorkflowFC4A11C3" + } + ], + "/aws-cdk-glue-workflow/MyWeeklyWorkflow/MyWeeklySchedule": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWeeklyWorkflowMyWeeklySchedule66671664" + } + ], + "/aws-cdk-glue-workflow/MyMonthlyWorkflow/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyMonthlyWorkflow5B4B0907" + } + ], + "/aws-cdk-glue-workflow/MyMonthlyWorkflow/MyMonthlySchedule": [ + { + "type": "aws:cdk:logicalId", + "data": "MyMonthlyWorkflowMyMonthlySchedule14F546AC" + } + ], + "/aws-cdk-glue-workflow/MyEventWorkflow/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyEventWorkflow0AA02AF3" + } + ], + "/aws-cdk-glue-workflow/MyEventWorkflow/MyNotifyEvent": [ + { + "type": "aws:cdk:logicalId", + "data": "MyEventWorkflowMyNotifyEvent4590A017" + } + ], + "/aws-cdk-glue-workflow/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/aws-cdk-glue-workflow/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "aws-cdk-glue-workflow" + }, + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json", + "terminationProtection": false, + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets" + ], + "metadata": { + "/aws-cdk-glue-table-integ/DefaultTest/DeployAssert/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/aws-cdk-glue-table-integ/DefaultTest/DeployAssert/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert" + }, + "Tree": { + "type": "cdk:tree", + "properties": { + "file": "tree.json" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/tree.json new file mode 100644 index 0000000000000..41f0f64c70296 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/tree.json @@ -0,0 +1,1090 @@ +{ + "version": "tree-0.1", + "tree": { + "id": "App", + "path": "", + "children": { + "aws-cdk-glue-workflow": { + "id": "aws-cdk-glue-workflow", + "path": "aws-cdk-glue-workflow", + "children": { + "MyWorkflowTask": { + "id": "MyWorkflowTask", + "path": "aws-cdk-glue-workflow/MyWorkflowTask", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyWorkflowTask/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "Role": { + "id": "Role", + "path": "aws-cdk-glue-workflow/Role", + "children": { + "ImportRole": { + "id": "ImportRole", + "path": "aws-cdk-glue-workflow/Role/ImportRole", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/Role/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "MyJob": { + "id": "MyJob", + "path": "aws-cdk-glue-workflow/MyJob", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-cdk-glue-workflow/MyJob/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-cdk-glue-workflow/MyJob/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyJob/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-cdk-glue-workflow/MyJob/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyJob/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "MyJobServiceRoleDefaultPolicy3BDBBD28", + "roles": [ + { + "Ref": "MyJobServiceRoleE6853162" + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "Code90269e4a92c3a84619f8820dfa764340": { + "id": "Code90269e4a92c3a84619f8820dfa764340", + "path": "aws-cdk-glue-workflow/MyJob/Code90269e4a92c3a84619f8820dfa764340", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-cdk-glue-workflow/MyJob/Code90269e4a92c3a84619f8820dfa764340/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-cdk-glue-workflow/MyJob/Code90269e4a92c3a84619f8820dfa764340/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyJob/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "pythonshell", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "defaultArguments": { + "--job-language": "python" + }, + "glueVersion": "1.0", + "role": { + "Fn::GetAtt": [ + "MyJobServiceRoleE6853162", + "Arn" + ] + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" + } + }, + "MyCrawler": { + "id": "MyCrawler", + "path": "aws-cdk-glue-workflow/MyCrawler", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Crawler", + "aws:cdk:cloudformation:props": { + "databaseName": "my_database", + "role": { + "Fn::GetAtt": [ + "Role1ABCC5F0", + "Arn" + ] + }, + "targets": { + "s3Targets": [ + { + "path": "s3://my_bucket" + } + ] + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnCrawler", + "version": "0.0.0" + } + }, + "MyPredicateJob": { + "id": "MyPredicateJob", + "path": "aws-cdk-glue-workflow/MyPredicateJob", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-cdk-glue-workflow/MyPredicateJob/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "MyPredicateJobServiceRoleDefaultPolicy87CB55E2", + "roles": [ + { + "Ref": "MyPredicateJobServiceRoleE1F838CC" + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyPredicateJob/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "pythonshell", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "defaultArguments": { + "--job-language": "python" + }, + "glueVersion": "1.0", + "role": { + "Fn::GetAtt": [ + "MyPredicateJobServiceRoleE1F838CC", + "Arn" + ] + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" + } + }, + "MyPredicateCrawler": { + "id": "MyPredicateCrawler", + "path": "aws-cdk-glue-workflow/MyPredicateCrawler", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Crawler", + "aws:cdk:cloudformation:props": { + "databaseName": "my_database", + "role": { + "Fn::GetAtt": [ + "Role1ABCC5F0", + "Arn" + ] + }, + "targets": { + "s3Targets": [ + { + "path": "s3://my_bucket" + } + ] + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnCrawler", + "version": "0.0.0" + } + }, + "MySecurityConfiguration": { + "id": "MySecurityConfiguration", + "path": "aws-cdk-glue-workflow/MySecurityConfiguration", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MySecurityConfiguration/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::SecurityConfiguration", + "aws:cdk:cloudformation:props": { + "encryptionConfiguration": { + "s3Encryptions": [ + { + "s3EncryptionMode": "SSE-S3" + } + ] + }, + "name": "awscdkglueworkflowMySecurityConfigurationBBF7B904" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnSecurityConfiguration", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.SecurityConfiguration", + "version": "0.0.0" + } + }, + "MyWorkflow": { + "id": "MyWorkflow", + "path": "aws-cdk-glue-workflow/MyWorkflow", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyWorkflow/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": { + "description": "my_on_demand_workflow_description", + "name": "my_on_demand_workflow" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + }, + "MyOnDemandTrigger": { + "id": "MyOnDemandTrigger", + "path": "aws-cdk-glue-workflow/MyWorkflow/MyOnDemandTrigger", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": {} + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": {} + } + ], + "name": "my_on_demand_trigger", + "type": "ON_DEMAND", + "workflowName": { + "Ref": "MyWorkflowBA72662B" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + }, + "MyConditionalTrigger": { + "id": "MyConditionalTrigger", + "path": "aws-cdk-glue-workflow/MyWorkflow/MyConditionalTrigger", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + } + ], + "name": "my_conditional_trigger", + "predicate": { + "logical": "AND", + "conditions": [ + { + "logical": "AND", + "jobName": { + "Ref": "MyPredicateJobF30A7B45" + }, + "state": "SUCCEEDED", + "logicalOperator": "EQUALS" + }, + { + "logical": "AND", + "crawlerName": { + "Ref": "MyPredicateCrawler" + }, + "crawlState": "SUCCEEDED", + "logicalOperator": "EQUALS" + } + ] + }, + "startOnCreation": true, + "type": "CONDITIONAL", + "workflowName": { + "Ref": "MyWorkflowBA72662B" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "MyDailyWorkflow": { + "id": "MyDailyWorkflow", + "path": "aws-cdk-glue-workflow/MyDailyWorkflow", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyDailyWorkflow/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": { + "description": "my_daily_workflow_description", + "name": "my_daily_workflow" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + }, + "MyDailySchedule": { + "id": "MyDailySchedule", + "path": "aws-cdk-glue-workflow/MyDailyWorkflow/MyDailySchedule", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + } + ], + "name": "my_daily_schedule", + "schedule": "cron(0 1 * * ? *)", + "startOnCreation": true, + "type": "SCHEDULED", + "workflowName": { + "Ref": "MyDailyWorkflow3684C8E1" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "MyWeeklyWorkflow": { + "id": "MyWeeklyWorkflow", + "path": "aws-cdk-glue-workflow/MyWeeklyWorkflow", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyWeeklyWorkflow/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": { + "description": "my_weekly_workflow_description", + "name": "my_weekly_workflow" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + }, + "MyWeeklySchedule": { + "id": "MyWeeklySchedule", + "path": "aws-cdk-glue-workflow/MyWeeklyWorkflow/MyWeeklySchedule", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + } + ], + "name": "my_weekly_schedule", + "schedule": "cron(0 1 ? * MON *)", + "startOnCreation": true, + "type": "SCHEDULED", + "workflowName": { + "Ref": "MyWeeklyWorkflowFC4A11C3" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "MyMonthlyWorkflow": { + "id": "MyMonthlyWorkflow", + "path": "aws-cdk-glue-workflow/MyMonthlyWorkflow", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyMonthlyWorkflow/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": { + "description": "my_monthly_workflow_description", + "name": "my_monthly_workflow" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + }, + "MyMonthlySchedule": { + "id": "MyMonthlySchedule", + "path": "aws-cdk-glue-workflow/MyMonthlyWorkflow/MyMonthlySchedule", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + } + ], + "name": "my_monthly_schedule", + "schedule": "cron(0 1 1 * ? *)", + "startOnCreation": true, + "type": "SCHEDULED", + "workflowName": { + "Ref": "MyMonthlyWorkflow5B4B0907" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "MyEventWorkflow": { + "id": "MyEventWorkflow", + "path": "aws-cdk-glue-workflow/MyEventWorkflow", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyEventWorkflow/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": { + "description": "my_event_workflow_description", + "name": "my_event_workflow" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + }, + "MyNotifyEvent": { + "id": "MyNotifyEvent", + "path": "aws-cdk-glue-workflow/MyEventWorkflow/MyNotifyEvent", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + } + ], + "eventBatchingCondition": { + "batchSize": 10, + "batchWindow": 600 + }, + "name": "my_notify_event", + "type": "EVENT", + "workflowName": { + "Ref": "MyEventWorkflow0AA02AF3" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "imported-workflow-name": { + "id": "imported-workflow-name", + "path": "aws-cdk-glue-workflow/imported-workflow-name", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "imported-workflow-arn": { + "id": "imported-workflow-arn", + "path": "aws-cdk-glue-workflow/imported-workflow-arn", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-cdk-glue-workflow/BootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-cdk-glue-workflow/CheckBootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" + } + }, + "aws-cdk-glue-table-integ": { + "id": "aws-cdk-glue-table-integ", + "path": "aws-cdk-glue-table-integ", + "children": { + "DefaultTest": { + "id": "DefaultTest", + "path": "aws-cdk-glue-table-integ/DefaultTest", + "children": { + "Default": { + "id": "Default", + "path": "aws-cdk-glue-table-integ/DefaultTest/Default", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "DeployAssert": { + "id": "DeployAssert", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert", + "children": { + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert/BootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert/CheckBootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", + "version": "0.0.0" + } + }, + "Tree": { + "id": "Tree", + "path": "Tree", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.App", + "version": "0.0.0" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts index 69506e643f676..f93fdf4754c1a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts @@ -48,15 +48,14 @@ const securityConfiguration = new glue.SecurityConfiguration(stack, 'MySecurityC }, }); -const workflow = new glue.Workflow(stack, 'MyWorkflow', { - workflowName: 'my_workflow', - description: 'my_workflow_description', +const onDemandWorkflow = new glue.Workflow(stack, 'MyWorkflow', { + workflowName: 'my_on_demand_workflow', + description: 'my_on_demand_workflow_description', }); -workflow.addOnDemandTrigger('MyOnDemandTrigger', { +onDemandWorkflow.addOnDemandTrigger('MyOnDemandTrigger', { triggerName: 'my_on_demand_trigger', description: 'my_on_demand_trigger_description', - enabled: true, actions: [ { job: glueJob, @@ -74,7 +73,7 @@ workflow.addOnDemandTrigger('MyOnDemandTrigger', { ], }); -workflow.addConditionalTrigger('MyConditionalTrigger', { +onDemandWorkflow.addConditionalTrigger('MyConditionalTrigger', { triggerName: 'my_conditional_trigger', description: 'my_conditional_trigger_description', enabled: true, @@ -106,7 +105,12 @@ workflow.addConditionalTrigger('MyConditionalTrigger', { ], }); -workflow.addDailyScheduleTrigger('MyDailySchedule', { +const dailyWorkflow = new glue.Workflow(stack, 'MyDailyWorkflow', { + workflowName: 'my_daily_workflow', + description: 'my_daily_workflow_description', +}); + +dailyWorkflow.addDailyScheduleTrigger('MyDailySchedule', { triggerName: 'my_daily_schedule', description: 'my_daily_schedule_description', enabled: true, @@ -129,7 +133,12 @@ workflow.addDailyScheduleTrigger('MyDailySchedule', { ], }); -workflow.addWeeklyScheduleTrigger('MyWeeklySchedule', { +const weeklyWorkflow = new glue.Workflow(stack, 'MyWeeklyWorkflow', { + workflowName: 'my_weekly_workflow', + description: 'my_weekly_workflow_description', +}); + +weeklyWorkflow.addWeeklyScheduleTrigger('MyWeeklySchedule', { triggerName: 'my_weekly_schedule', description: 'my_weekly_schedule_description', enabled: true, @@ -152,7 +161,12 @@ workflow.addWeeklyScheduleTrigger('MyWeeklySchedule', { ], }); -workflow.addMonthlyScheduleTrigger('MyMonthlySchedule', { +const monthlyWorkflow = new glue.Workflow(stack, 'MyMonthlyWorkflow', { + workflowName: 'my_monthly_workflow', + description: 'my_monthly_workflow_description', +}); + +monthlyWorkflow.addMonthlyScheduleTrigger('MyMonthlySchedule', { triggerName: 'my_monthly_schedule', description: 'my_monthly_schedule_description', enabled: true, @@ -175,10 +189,14 @@ workflow.addMonthlyScheduleTrigger('MyMonthlySchedule', { ], }); -workflow.addNotifyEventTrigger('MyNotifyEvent', { +const eventWorkflow = new glue.Workflow(stack, 'MyEventWorkflow', { + workflowName: 'my_event_workflow', + description: 'my_event_workflow_description', +}); + +eventWorkflow.addNotifyEventTrigger('MyNotifyEvent', { triggerName: 'my_notify_event', description: 'my_notify_event_description', - enabled: true, batchSize: 10, batchWindow: cdk.Duration.minutes(10), actions: [ @@ -200,7 +218,7 @@ workflow.addNotifyEventTrigger('MyNotifyEvent', { ], }); -const importedWorkflowByName = glue.Workflow.fromWorkflowName(stack, 'imported-workflow-name', workflow.workflowName); +const importedWorkflowByName = glue.Workflow.fromWorkflowName(stack, 'imported-workflow-name', monthlyWorkflow.workflowName); glue.Workflow.fromWorkflowArn(stack, 'imported-workflow-arn', importedWorkflowByName.workflowArn); From 7f2386b768080f0bc0a6ad761f05788280a443c2 Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Mon, 5 Aug 2024 11:42:01 +0000 Subject: [PATCH 16/18] removing `this` keyword --- packages/@aws-cdk/aws-glue-alpha/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index ad623845cedd1..411f12f71f41a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -625,7 +625,7 @@ declare const myJob: glue.IJob; declare const myCrawler: glueCfn.CfnCrawler; declare const securityConfiguration: glue.ISecurityConfiguration; -myWorkflow.addOnDemandTrigger(this, 'OnDemandTrigger', { +myWorkflow.addOnDemandTrigger('OnDemandTrigger', { triggerName: 'on_demand_trigger', description: 'description', enabled: true, @@ -658,7 +658,7 @@ declare const myCrawler: glueCfn.CfnCrawler; declare const securityConfiguration: glue.ISecurityConfiguration; declare const schedule: events.Schedule; -myWorkflow.addCustomScheduleTrigger(this, 'ScheduleTrigger', { +myWorkflow.addCustomScheduleTrigger('ScheduleTrigger', { triggerName: 'schedule_trigger', description: 'description', enabled: true, @@ -689,7 +689,7 @@ declare const myJob: glue.IJob; declare const myCrawler: glueCfn.CfnCrawler; declare const securityConfiguration: glue.ISecurityConfiguration; -myWorkflow.addDailyScheduleTrigger(this, 'DailyScheduleTrigger', { +myWorkflow.addDailyScheduleTrigger('DailyScheduleTrigger', { triggerName: 'daily_schedule_trigger', description: 'description', enabled: true, @@ -710,7 +710,7 @@ myWorkflow.addDailyScheduleTrigger(this, 'DailyScheduleTrigger', { ], }); -myWorkflow.addWeeklyScheduleTrigger(this, 'WeeklyScheduleTrigger', { +myWorkflow.addWeeklyScheduleTrigger('WeeklyScheduleTrigger', { triggerName: 'weekly_schedule_trigger', description: 'description', enabled: true, @@ -731,7 +731,7 @@ myWorkflow.addWeeklyScheduleTrigger(this, 'WeeklyScheduleTrigger', { ], }); -myWorkflow.addMonthlyScheduleTrigger(this, 'MonthlyScheduleTrigger', { +myWorkflow.addMonthlyScheduleTrigger('MonthlyScheduleTrigger', { triggerName: 'monthly_schedule_trigger', description: 'description', enabled: true, @@ -763,7 +763,7 @@ declare const myJob: glue.IJob; declare const myCrawler: glueCfn.CfnCrawler; declare const securityConfiguration: glue.ISecurityConfiguration; -myWorkflow.addNotifyEventTrigger(this, 'EventTrigger', { +myWorkflow.addNotifyEventTrigger('EventTrigger', { triggerName: 'event_trigger', description: 'description', enabled: true, @@ -799,7 +799,7 @@ declare const predicateJob: glue.IJob; declare const predicateCrawler: glueCfn.CfnCrawler; declare const securityConfiguration: glue.ISecurityConfiguration; -myWorkflow.addConditionalTrigger(this, 'ConditionalTrigger', { +myWorkflow.addConditionalTrigger('ConditionalTrigger', { triggerName: 'conditional_trigger', description: 'description', enabled: true, From f50702fc34bf001491859305766c0e521128ac82 Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Mon, 5 Aug 2024 11:47:34 +0000 Subject: [PATCH 17/18] removing enabled fiedl --- packages/@aws-cdk/aws-glue-alpha/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index 411f12f71f41a..00c5cee28765f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -628,7 +628,6 @@ declare const securityConfiguration: glue.ISecurityConfiguration; myWorkflow.addOnDemandTrigger('OnDemandTrigger', { triggerName: 'on_demand_trigger', description: 'description', - enabled: true, actions: [ { job: myJob, @@ -766,7 +765,6 @@ declare const securityConfiguration: glue.ISecurityConfiguration; myWorkflow.addNotifyEventTrigger('EventTrigger', { triggerName: 'event_trigger', description: 'description', - enabled: true, batchSize: 10, batchWindow: cdk.Duration.minutes(5), actions: [ From 0740c891d6bc6a11e5c9f8ae60afb8a5b88f8005 Mon Sep 17 00:00:00 2001 From: Rizxcviii Date: Mon, 5 Aug 2024 13:40:41 +0000 Subject: [PATCH 18/18] functions now return triggers --- .../@aws-cdk/aws-glue-alpha/lib/workflow.ts | 162 ++++++++---------- 1 file changed, 73 insertions(+), 89 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts index 1094a85d08a95..4b9be8024c7e5 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts @@ -238,7 +238,7 @@ export interface IWorkflow extends IResource { * @param id The identifier for the trigger. * @param props The properties for the trigger. */ - addOnDemandTrigger(id: string, props: TriggerProps): void; + addOnDemandTrigger(id: string, props: TriggerProps): CfnTrigger; /** * Adds a trigger to the workflow to run on a daily schedule. @@ -246,7 +246,7 @@ export interface IWorkflow extends IResource { * @param id The identifier for the trigger. * @param props The properties for the trigger. */ - addDailyScheduleTrigger(id: string, props: ActivatableTriggerProps): void; + addDailyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger; /** * Adds a trigger to the workflow to run on a weekly schedule. @@ -254,7 +254,7 @@ export interface IWorkflow extends IResource { * @param id The identifier for the trigger. * @param props The properties for the trigger. */ - addWeeklyScheduleTrigger(id: string, props: ActivatableTriggerProps): void; + addWeeklyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger; /** * Adds a trigger to the workflow to run on a monthly schedule. @@ -262,7 +262,7 @@ export interface IWorkflow extends IResource { * @param id The identifier for the trigger. * @param props The properties for the trigger. */ - addMonthlyScheduleTrigger(id: string, props: ActivatableTriggerProps): void; + addMonthlyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger; /** * Adds a trigger to the workflow to run on a custom schedule. @@ -270,7 +270,7 @@ export interface IWorkflow extends IResource { * @param id The identifier for the trigger. * @param props The properties for the trigger. */ - addCustomScheduleTrigger(id: string, props: ScheduleTriggerProps): void; + addCustomScheduleTrigger(id: string, props: ScheduleTriggerProps): CfnTrigger; /** * Adds a trigger to the workflow to run on a notification event. @@ -278,7 +278,7 @@ export interface IWorkflow extends IResource { * @param id The identifier for the trigger. * @param props The properties for the trigger. */ - addNotifyEventTrigger(id: string, props: NotificationTriggerProps): void; + addNotifyEventTrigger(id: string, props: NotificationTriggerProps): CfnTrigger; /** * Adds a trigger to the workflow to run on a conditional event. @@ -286,7 +286,7 @@ export interface IWorkflow extends IResource { * @param id The identifier for the trigger. * @param props The properties for the trigger. */ - addConditionalTrigger(id: string, props: ConditionalTriggerProps): void; + addConditionalTrigger(id: string, props: ConditionalTriggerProps): CfnTrigger; } /** @@ -333,108 +333,94 @@ abstract class WorkflowBase extends Resource implements IWorkflow { */ public abstract readonly workflowName: string; - private readonly triggers: CfnTrigger[] = []; - /** * Adds a trigger to the workflow to run on demand. */ - public addOnDemandTrigger(id: string, props: TriggerProps): void { - this.triggers.push( - new CfnTrigger(this, id, { - name: props.triggerName, - type: 'ON_DEMAND', - workflowName: this.workflowName, - actions: props.actions?.map(action => this.renderAction(action)), - }), - ); + public addOnDemandTrigger(id: string, props: TriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'ON_DEMAND', + workflowName: this.workflowName, + actions: props.actions?.map(action => this.renderAction(action)), + }); } /** * Adds a trigger to the workflow to run on a daily schedule. */ - public addDailyScheduleTrigger(id: string, props: ActivatableTriggerProps): void { - this.triggers.push( - new CfnTrigger(this, id, { - name: props.triggerName, - type: 'SCHEDULED', - schedule: 'cron(0 1 * * ? *)', - workflowName: this.workflowName, - startOnCreation: props.enabled ?? true, - actions: props.actions?.map(action => this.renderAction(action)), - }), - ); + public addDailyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'SCHEDULED', + schedule: 'cron(0 1 * * ? *)', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)), + }); } /** * Adds a trigger to the workflow to run on a weekly schedule. */ - public addWeeklyScheduleTrigger(id: string, props: ActivatableTriggerProps): void { - this.triggers.push( - new CfnTrigger(this, id, { - name: props.triggerName, - type: 'SCHEDULED', - schedule: 'cron(0 1 ? * MON *)', - workflowName: this.workflowName, - startOnCreation: props.enabled ?? true, - actions: props.actions?.map(action => this.renderAction(action)), - }), - ); + public addWeeklyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'SCHEDULED', + schedule: 'cron(0 1 ? * MON *)', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)), + }); } /** * Adds a trigger to the workflow to run on a monthly schedule. */ - public addMonthlyScheduleTrigger(id: string, props: ActivatableTriggerProps): void { - this.triggers.push( - new CfnTrigger(this, id, { - name: props.triggerName, - type: 'SCHEDULED', - schedule: 'cron(0 1 1 * ? *)', - workflowName: this.workflowName, - startOnCreation: props.enabled ?? true, - actions: props.actions?.map(action => this.renderAction(action)) ?? [], - }), - ); + public addMonthlyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'SCHEDULED', + schedule: 'cron(0 1 1 * ? *)', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)) ?? [], + }); } /** * Adds a trigger to the workflow to run on a custom schedule. */ - public addCustomScheduleTrigger(id: string, props: ScheduleTriggerProps): void { - this.triggers.push( - new CfnTrigger(this, id, { - name: props.triggerName, - type: 'SCHEDULED', - schedule: props.schedule.expressionString, - workflowName: this.workflowName, - startOnCreation: props.enabled ?? true, - actions: props.actions?.map(action => this.renderAction(action)) ?? [], - }), - ); + public addCustomScheduleTrigger(id: string, props: ScheduleTriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'SCHEDULED', + schedule: props.schedule.expressionString, + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)) ?? [], + }); } /** * Adds a trigger to the workflow to run on a notification event. */ - public addNotifyEventTrigger(id: string, props: NotificationTriggerProps): void { - this.triggers.push( - new CfnTrigger(this, id, { - name: props.triggerName, - type: 'EVENT', - workflowName: this.workflowName, - actions: props.actions?.map(action => this.renderAction(action)) ?? [], - eventBatchingCondition: { - batchSize: props.batchSize ?? 1, - batchWindow: props.batchWindow?.toSeconds() ?? 900, - }, - }), - ); + public addNotifyEventTrigger(id: string, props: NotificationTriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'EVENT', + workflowName: this.workflowName, + actions: props.actions?.map(action => this.renderAction(action)) ?? [], + eventBatchingCondition: { + batchSize: props.batchSize ?? 1, + batchWindow: props.batchWindow?.toSeconds() ?? 900, + }, + }); } /** * Adds a trigger to the workflow to run on a conditional event. */ - public addConditionalTrigger(id: string, props: ConditionalTriggerProps): void { + public addConditionalTrigger(id: string, props: ConditionalTriggerProps): CfnTrigger { this.validatePredicates(props.jobPredicates, props.crawlerPredicates); const conditions: CfnTrigger.ConditionProperty[] = []; @@ -459,19 +445,17 @@ abstract class WorkflowBase extends Resource implements IWorkflow { ); } - this.triggers.push( - new CfnTrigger(this, id, { - name: props.triggerName, - type: 'CONDITIONAL', - workflowName: this.workflowName, - startOnCreation: props.enabled ?? true, - actions: props.actions?.map(action => this.renderAction(action)), - predicate: { - logical: props.predicateCondition, - conditions: conditions.length > 0 ? conditions : undefined, - }, - }), - ); + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'CONDITIONAL', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)), + predicate: { + logical: props.predicateCondition, + conditions: conditions.length > 0 ? conditions : undefined, + }, + }); } protected renderAction(action: Action): CfnTrigger.ActionProperty {