From 9e623ef8f9e95b037595f16bd5611ca1fb52ca49 Mon Sep 17 00:00:00 2001 From: addis-samtek <155009976+addis-samtek@users.noreply.github.com> Date: Tue, 19 Nov 2024 00:16:55 -0500 Subject: [PATCH 01/10] Add text extract stack cdk --- .github/workflows/deploy-experimental.yml | 43 +++ cdk-eregs/bin/cdk-eregs.ts | 27 +- cdk-eregs/lib/stacks/text-extract-stack.ts | 360 +++++++++++++++++++++ 3 files changed, 429 insertions(+), 1 deletion(-) create mode 100644 cdk-eregs/lib/stacks/text-extract-stack.ts diff --git a/.github/workflows/deploy-experimental.yml b/.github/workflows/deploy-experimental.yml index 5e68cfc33..31d010ae0 100644 --- a/.github/workflows/deploy-experimental.yml +++ b/.github/workflows/deploy-experimental.yml @@ -201,6 +201,49 @@ jobs: -c environment=${{ env.ENVIRONMENT_NAME }} \ --require-approval never popd + deploy-Text-extract-CDK: + environment: + name: "dev" + env: + ENVIRONMENT_NAME: "dev" + runs-on: ubuntu-22.04 + steps: + # Checkout the code + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + # Find the PR number. This is not always trivial which is why this uses an existign action + - name: Find PR number + uses: jwalton/gh-find-current-pr@v1 + id: findPr + with: + # Can be "open", "closed", or "all". Defaults to "open". + state: open + # Configure AWS credentials for GitHub Actions + - name: Configure AWS credentials for GitHub Actions + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_TO_ASSUME }} + aws-region: us-east-1 + - name: Deploy Text extract via CDK + id: deploy-text-extract + if: success() && steps.findPr.outputs.number + env: + PR_NUMBER: ${{ steps.findPr.outputs.pr }} + RUN_ID: ${{ github.run_id }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + CDK_DEBUG: true + ENVIRONMENT_NAME: ${{ env.ENVIRONMENT_NAME }} + run: | + pushd cdk-eregs + npm install -g aws-cdk@latest @aws-sdk/client-ssm + npm install + cdk deploy "*-text-extractor" \ + -c environment=${{ env.ENVIRONMENT_NAME }} \ + --require-approval never + popd deploy-text-extractor: environment: name: "dev" diff --git a/cdk-eregs/bin/cdk-eregs.ts b/cdk-eregs/bin/cdk-eregs.ts index 42b8cef2d..c03f5edfa 100644 --- a/cdk-eregs/bin/cdk-eregs.ts +++ b/cdk-eregs/bin/cdk-eregs.ts @@ -9,7 +9,7 @@ import { IamPermissionsBoundaryAspect } from '../lib/aspects/iam-permissions-bou import { EphemeralRemovalPolicyAspect } from '../lib/aspects/removal-policy-aspect'; import * as lambda from 'aws-cdk-lib/aws-lambda'; import { MaintenanceApiStack } from '../lib/stacks/maintainance-stack'; - +import { TextExtractorStack } from '../lib/stacks/text-extract-stack'; // import { StackFactory } from '../lib/factories/stack-factory'; // import { getStackConfigs } from '../config/stack-definition'; async function main() { @@ -17,6 +17,17 @@ async function main() { // const synthesizerConfigJson = await getParameterValue('/cms/cloud/cdkSynthesizerConfig'); const synthesizerConfigJson = await getParameterValue('/eregulations/cdk_config'); const synthesizerConfig = JSON.parse(synthesizerConfigJson); + const [vpcId, logLevel, httpUser, httpPassword] = await Promise.all([ + getParameterValue('/account_vars/vpc/id'), + getParameterValue('/eregulations/text_extractor/log_level'), + getParameterValue('/eregulations/http/user'), + getParameterValue('/eregulations/http/password'), + ]); + // Get environment configuration + const env = { + account: process.env.CDK_DEFAULT_ACCOUNT || process.env.AWS_ACCOUNT_ID, + region: process.env.CDK_DEFAULT_REGION || 'us-east-1' + }; const app = new cdk.App({ defaultStackSynthesizer: new cdk.DefaultStackSynthesizer(synthesizerConfig), }); @@ -100,6 +111,20 @@ async function main() { loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, }, }, stageConfig); + new TextExtractorStack(app, stageConfig.getResourceName('text-extractor'), { + env, + lambdaConfig: { + memorySize: 1024, + timeout: 900, + reservedConcurrentExecutions: 10, + }, + environmentConfig: { + vpcId, + logLevel, + httpUser, + httpPassword, + } + }, stageConfig); // Example deployment in app.ts // Apply aspects diff --git a/cdk-eregs/lib/stacks/text-extract-stack.ts b/cdk-eregs/lib/stacks/text-extract-stack.ts new file mode 100644 index 000000000..1f40cd03a --- /dev/null +++ b/cdk-eregs/lib/stacks/text-extract-stack.ts @@ -0,0 +1,360 @@ +// lib/stacks/text-extractor-stack.ts +import * as cdk from 'aws-cdk-lib'; +import { + aws_iam as iam, + aws_logs as logs, + aws_lambda as lambda, + aws_sqs as sqs, + aws_ec2 as ec2, +} from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import { StageConfig } from '../../config/stage-config'; +import * as path from 'path'; + +/** + * Configuration interface for Lambda function settings. + * Defines required and optional parameters for Lambda function configuration. + */ +interface LambdaConfig { + /** Memory allocation in MB for the Lambda function */ + memorySize: number; + /** Function timeout in seconds */ + timeout: number; + /** Optional limit on concurrent executions */ + reservedConcurrentExecutions?: number; +} + +/** + * Configuration interface for environment-specific settings. + * Contains all the external configuration values needed for the stack. + */ +interface EnvironmentConfig { + /** VPC ID where the Lambda function will be deployed */ + vpcId: string; + /** Log level for the Lambda function (e.g., DEBUG, INFO) */ + logLevel: string; + /** HTTP basic auth username for API authentication */ + httpUser: string; + /** HTTP basic auth password for API authentication */ + httpPassword: string; +} + +/** + * Properties interface for TextExtractorStack. + * Extends standard CDK stack properties with custom configuration settings. + */ +export interface TextExtractorStackProps extends cdk.StackProps { + /** Lambda function configuration settings */ + lambdaConfig: LambdaConfig; + /** Environment-specific configuration settings */ + environmentConfig: EnvironmentConfig; +} + +/** + * CDK Stack implementation for Text Extractor service. + * + * This stack creates a serverless text extraction service with the following components: + * - Docker-based Lambda function for text extraction + * - SQS Queue with Dead Letter Queue for reliable message processing + * - IAM roles and policies for secure access + * - CloudWatch Log Groups for monitoring + * - VPC Security Group for network isolation + * - Event Source Mapping for queue integration + * + * The stack is designed to be environment-aware and supports deployment across + * different stages (dev, staging, prod) through the StageConfig parameter. + * + * @example + * ```typescript + * // Create the stack with environment configuration + * new TextExtractorStack(app, 'text-extractor', { + * env: { account: '123456789', region: 'us-east-1' }, + * lambdaConfig: { + * memorySize: 1024, + * timeout: 900, + * reservedConcurrentExecutions: 10, + * }, + * environmentConfig: { + * vpcId: 'vpc-123456', + * logLevel: 'DEBUG', + * httpUser: 'user', + * httpPassword: 'pass' + * } + * }, stageConfig); + * ``` + */ +export class TextExtractorStack extends cdk.Stack { + /** The Lambda function that processes text extraction requests */ + public readonly lambda: lambda.Function; + /** The SQS queue that holds text extraction requests */ + public readonly queue: sqs.Queue; + /** The Dead Letter Queue for failed message processing */ + private readonly deadLetterQueue: sqs.Queue; + /** Stage configuration for environment-aware deployments */ + private readonly stageConfig: StageConfig; + + /** + * Creates a new instance of TextExtractorStack. + * + * @param scope - The scope in which to define this construct + * @param id - The scoped construct ID + * @param props - Configuration properties for the stack + * @param stageConfig - Environment stage configuration + */ + constructor( + scope: Construct, + id: string, + props: TextExtractorStackProps, + stageConfig: StageConfig + ) { + super(scope, id, props); + this.stageConfig = stageConfig; + + // Create VPC reference + const vpc = ec2.Vpc.fromLookup(this, 'VPC', { + vpcId: props.environmentConfig.vpcId + }); + + // Create security group + const securityGroup = new ec2.SecurityGroup(this, 'ServerlessSecurityGroup', { + vpc, + description: 'SecurityGroup for Serverless Functions', + allowAllOutbound: true, + }); + + // Create DLQ + this.deadLetterQueue = new sqs.Queue(this, 'DeadLetterQueue', { + queueName: this.stageConfig.getResourceName('text-extractor-dl-queue'), + }); + + // Create main queue + this.queue = new sqs.Queue(this, 'TextExtractorQueue', { + queueName: this.stageConfig.getResourceName('text-extractor-queue'), + visibilityTimeout: cdk.Duration.seconds(900), + retentionPeriod: cdk.Duration.days(4), + deadLetterQueue: { + maxReceiveCount: 5, + queue: this.deadLetterQueue, + }, + }); + + // Create Lambda infrastructure + const { lambdaRole, logGroup } = this.createLambdaInfrastructure(); + + // Create Lambda function + this.lambda = this.createLambdaFunction( + props.lambdaConfig, + props.environmentConfig, + lambdaRole, + vpc, + securityGroup + ); + + // Create event source mapping + this.createEventSourceMapping(); + + // Create stack outputs + this.createStackOutputs(); + } + + /** + * Creates and configures the Lambda function with Docker image. + * + * @param config - Lambda function configuration + * @param envConfig - Environment-specific configuration + * @param role - IAM role for the function + * @param vpc - VPC where the function will be deployed + * @param securityGroup - Security group for the function + * @returns Configured Lambda function + * @private + */ + private createLambdaFunction( + config: LambdaConfig, + envConfig: EnvironmentConfig, + role: iam.Role, + vpc: ec2.IVpc, + securityGroup: ec2.SecurityGroup, + ): lambda.Function { + const dockerContextPath = path.resolve(__dirname, '../../../solution/text-extractor/'); + console.log('Docker context path:', dockerContextPath); + + return new lambda.DockerImageFunction(this, 'TextExtractorFunction', { + functionName: this.stageConfig.getResourceName('text-extractor'), + code: lambda.DockerImageCode.fromImageAsset(dockerContextPath, { + file: 'Dockerfile', + }), + vpc, + securityGroups: [securityGroup], + memorySize: config.memorySize, + timeout: cdk.Duration.seconds(config.timeout), + reservedConcurrentExecutions: config.reservedConcurrentExecutions, + environment: { + LOG_LEVEL: envConfig.logLevel, + HTTP_AUTH_USER: envConfig.httpUser, + HTTP_AUTH_PASSWORD: envConfig.httpPassword, + }, + role, + }); + } + + /** + * Creates event source mapping between SQS queue and Lambda function. + * Configures single-message processing with batch size of 1. + * + * @private + */ + private createEventSourceMapping() { + new lambda.EventSourceMapping(this, 'TextExtractorEventSourceMapping', { + target: this.lambda, + batchSize: 1, + eventSourceArn: this.queue.queueArn, + enabled: true, + }); + } + + /** + * Creates Lambda infrastructure including IAM role and log group. + * Sets up: + * - CloudWatch Log Group with infinite retention + * - IAM Role with proper permissions + * - Required IAM policies for Queue, Textract, and S3 access + * + * @returns Object containing IAM role and Log Group + * @private + */ + private createLambdaInfrastructure() { + const logGroup = new logs.LogGroup(this, 'TextExtractorLogGroup', { + logGroupName: this.stageConfig.aws.lambda('text-extractor'), + retention: logs.RetentionDays.INFINITE, + }); + + const lambdaRole = new iam.Role(this, 'LambdaFunctionRole', { + path: this.stageConfig.iamPath, + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + permissionsBoundary: iam.ManagedPolicy.fromManagedPolicyArn( + this, + 'PermissionsBoundary', + this.stageConfig.permissionsBoundaryArn + ), + managedPolicies: [ + iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaVPCAccessExecutionRole'), + ], + inlinePolicies: { + QueuePolicy: this.createQueuePolicy(), + TextDetectionPolicy: this.createTextDetectionPolicy(), + LambdaPolicy: this.createLambdaPolicy(), + }, + }); + + return { lambdaRole, logGroup }; + } + + /** + * Creates SQS queue access policy. + * Grants permissions for message reception and deletion. + * + * @returns IAM policy document for queue access + * @private + */ + private createQueuePolicy(): iam.PolicyDocument { + return new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'sqs:ReceiveMessage', + 'sqs:DeleteMessage', + 'sqs:GetQueueAttributes' + ], + resources: [this.queue.queueArn], + }), + ], + }); + } + + /** + * Creates Amazon Textract service access policy. + * Grants permission to use the DetectDocumentText API. + * + * @returns IAM policy document for Textract access + * @private + */ + private createTextDetectionPolicy(): iam.PolicyDocument { + return new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + sid: 'DetectDocumentText', + effect: iam.Effect.ALLOW, + actions: ['textract:DetectDocumentText'], + resources: ['*'], + }), + ], + }); + } + + /** + * Creates Lambda execution policy for logs and S3 access. + * Grants permissions for: + * - CloudWatch Logs creation and management + * - S3 object operations on the environment-specific bucket + * + * @returns IAM policy document for Lambda execution + * @private + */ + private createLambdaPolicy(): iam.PolicyDocument { + return new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'logs:CreateLogGroup', + 'logs:CreateLogStream', + 'logs:PutLogEvents' + ], + resources: [`arn:aws:logs:${this.region}:${this.account}:log-group:/aws/lambda/*:*:*`], + }), + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 's3:PutObject', + 's3:GetObject', + 's3:DeleteObject' + ], + resources: [ + `arn:aws:s3:::file-repo-eregs-${this.stageConfig.environment}*` + ], + }), + ], + }); + } + + /** + * Creates CloudFormation outputs for stack resources. + * Exports: + * - Lambda function ARN + * - SQS queue URL and ARN + * These outputs can be imported by other stacks or used for external reference. + * + * @private + */ + private createStackOutputs() { + const outputs: Record = { + TextExtractorLambdaFunctionQualifiedArn: { + value: this.lambda.functionArn, + description: 'Current Lambda function version', + exportName: this.stageConfig.getResourceName('text-extractor-lambda-arn'), + }, + TextExtractorQueueUrl: { + value: this.queue.queueUrl, + exportName: this.stageConfig.getResourceName('text-extractor-queue-url'), + }, + TextExtractorQueueArn: { + value: this.queue.queueArn, + exportName: this.stageConfig.getResourceName('text-extractor-queue-arn'), + } + }; + + Object.entries(outputs).forEach(([name, props]) => new cdk.CfnOutput(this, name, props)); + } +} + From 38994242fc50b9bc4683474014ae53650f4e0cee Mon Sep 17 00:00:00 2001 From: addis-samtek <155009976+addis-samtek@users.noreply.github.com> Date: Tue, 19 Nov 2024 22:16:01 -0500 Subject: [PATCH 02/10] Add text extract stack cdk --- cdk-eregs/cdk.context.json | 44 ++++++++++++++++++++++ cdk-eregs/lib/stacks/text-extract-stack.ts | 4 +- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/cdk-eregs/cdk.context.json b/cdk-eregs/cdk.context.json index 6e307106c..c844fd1a6 100644 --- a/cdk-eregs/cdk.context.json +++ b/cdk-eregs/cdk.context.json @@ -62,5 +62,49 @@ ] } ] + }, + "vpc-provider:account=910670052382:filter.vpc-id=vpc-0b4ec773c3a0aff85:region=us-east-1:returnAsymmetricSubnets=true": { + "vpcId": "vpc-0b4ec773c3a0aff85", + "vpcCidrBlock": "10.147.73.192/26", + "ownerAccountId": "910670052382", + "availabilityZones": [], + "subnetGroups": [ + { + "name": "Public", + "type": "Public", + "subnets": [ + { + "subnetId": "subnet-06a498eab225d421f", + "cidr": "10.147.73.192/28", + "availabilityZone": "us-east-1a", + "routeTableId": "rtb-0319ac9f2e2587861" + }, + { + "subnetId": "subnet-045ffa689dee9bbaf", + "cidr": "10.147.73.224/28", + "availabilityZone": "us-east-1a", + "routeTableId": "rtb-0f86dd33477a534a3" + }, + { + "subnetId": "subnet-086a464d9c0e8ec4a", + "cidr": "10.147.73.240/28", + "availabilityZone": "us-east-1b", + "routeTableId": "rtb-0f86dd33477a534a3" + } + ] + }, + { + "name": "Private", + "type": "Private", + "subnets": [ + { + "subnetId": "subnet-020cc2513ecb5e5e4", + "cidr": "10.147.73.208/28", + "availabilityZone": "us-east-1b", + "routeTableId": "rtb-0a8ad3bde1bfa6ba8" + } + ] + } + ] } } diff --git a/cdk-eregs/lib/stacks/text-extract-stack.ts b/cdk-eregs/lib/stacks/text-extract-stack.ts index 1f40cd03a..b48951549 100644 --- a/cdk-eregs/lib/stacks/text-extract-stack.ts +++ b/cdk-eregs/lib/stacks/text-extract-stack.ts @@ -142,7 +142,7 @@ export class TextExtractorStack extends cdk.Stack { const { lambdaRole, logGroup } = this.createLambdaInfrastructure(); // Create Lambda function - this.lambda = this.createLambdaFunction( + this.lambda = this.createTextExtractorLambdaFunction( props.lambdaConfig, props.environmentConfig, lambdaRole, @@ -168,7 +168,7 @@ export class TextExtractorStack extends cdk.Stack { * @returns Configured Lambda function * @private */ - private createLambdaFunction( + private createTextExtractorLambdaFunction( config: LambdaConfig, envConfig: EnvironmentConfig, role: iam.Role, From 9eaf918e4fa67d7a303018447a3c372a61d38162 Mon Sep 17 00:00:00 2001 From: addis-samtek <155009976+addis-samtek@users.noreply.github.com> Date: Tue, 19 Nov 2024 22:40:11 -0500 Subject: [PATCH 03/10] Add text extract stack cdk --- .github/workflows/deploy-experimental.yml | 6 +++--- cdk-eregs/bin/cdk-eregs.ts | 22 +++++++++------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy-experimental.yml b/.github/workflows/deploy-experimental.yml index 31d010ae0..378afc063 100644 --- a/.github/workflows/deploy-experimental.yml +++ b/.github/workflows/deploy-experimental.yml @@ -156,7 +156,7 @@ jobs: npm install cdk deploy "*redirect-api" \ -c environment=${{ env.ENVIRONMENT_NAME }} \ - --require-approval never + --require-approval never --tags lambda-type=zip popd deploy-Maintenance-Api-CDK: environment: @@ -199,7 +199,7 @@ jobs: npm install cdk deploy "*maintenance-api" \ -c environment=${{ env.ENVIRONMENT_NAME }} \ - --require-approval never + --require-approval never --tags lambda-type=zip popd deploy-Text-extract-CDK: environment: @@ -242,7 +242,7 @@ jobs: npm install cdk deploy "*-text-extractor" \ -c environment=${{ env.ENVIRONMENT_NAME }} \ - --require-approval never + --require-approval never --tags lambda-type=docker popd deploy-text-extractor: environment: diff --git a/cdk-eregs/bin/cdk-eregs.ts b/cdk-eregs/bin/cdk-eregs.ts index c03f5edfa..8096c4007 100644 --- a/cdk-eregs/bin/cdk-eregs.ts +++ b/cdk-eregs/bin/cdk-eregs.ts @@ -31,7 +31,8 @@ async function main() { const app = new cdk.App({ defaultStackSynthesizer: new cdk.DefaultStackSynthesizer(synthesizerConfig), }); - + // Get bundling type from context + const bundlingType = app.node.tryGetContext('bundling'); const environment = app.node.tryGetContext('environment') || process.env.DEPLOY_ENV || process.env.GITHUB_JOB_ENVIRONMENT || @@ -82,16 +83,8 @@ async function main() { cdk.Tags.of(app).add(key, value); }); - // // Create and configure stacks - // const stackFactory = new StackFactory(app, stageConfig); - // const stackConfigs = getStackConfigs(environment, stageConfig.isEphemeral()); - - // const stacks = stackConfigs - // .filter(config => config.enabled) - // .map(config => stackFactory.createStack(config)) - // .filter((stack): stack is cdk.Stack => stack !== null); // Create RedirectApiStack - new RedirectApiStack(app, stageConfig.getResourceName('redirect-api'), { + const redirectStack = new RedirectApiStack(app, stageConfig.getResourceName('redirect-api'), { lambdaConfig: { runtime: lambda.Runtime.PYTHON_3_12, memorySize: 1024, @@ -101,7 +94,7 @@ async function main() { loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, }, }, stageConfig); - new MaintenanceApiStack(app, stageConfig.getResourceName('maintenance-api'), { + const maintenanceStack = new MaintenanceApiStack(app, stageConfig.getResourceName('maintenance-api'), { lambdaConfig: { runtime: lambda.Runtime.PYTHON_3_12, memorySize: 1024, @@ -111,7 +104,9 @@ async function main() { loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, }, }, stageConfig); - new TextExtractorStack(app, stageConfig.getResourceName('text-extractor'), { + cdk.Tags.of(redirectStack).add('lambda-type', 'zip'); + cdk.Tags.of(maintenanceStack).add('lambda-type', 'zip'); + const textExtractorStack = new TextExtractorStack(app, stageConfig.getResourceName('text-extractor'), { env, lambdaConfig: { memorySize: 1024, @@ -125,7 +120,8 @@ async function main() { httpPassword, } }, stageConfig); - // Example deployment in app.ts + cdk.Tags.of(textExtractorStack).add('lambda-type', 'docker'); + // Apply aspects await applyGlobalAspects(app, stageConfig); From 80d586121dde6908f9d1e6cfeb1ab3eacdac5612 Mon Sep 17 00:00:00 2001 From: addis-samtek <155009976+addis-samtek@users.noreply.github.com> Date: Tue, 19 Nov 2024 23:18:18 -0500 Subject: [PATCH 04/10] Add text extract stack cdk --- cdk-eregs/lib/stacks/redirect-stack.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cdk-eregs/lib/stacks/redirect-stack.ts b/cdk-eregs/lib/stacks/redirect-stack.ts index fa84a607d..96f44c199 100644 --- a/cdk-eregs/lib/stacks/redirect-stack.ts +++ b/cdk-eregs/lib/stacks/redirect-stack.ts @@ -109,7 +109,9 @@ export class RedirectApiStack extends cdk.Stack { description: `Redirect API Lambda function for ${this.stageConfig.environment} stage`, runtime: config.runtime, handler: config.handler ?? 'redirect_lambda.handler', - code: lambda.Code.fromAsset(config.codePath ?? '../solution/backend/'), + code: lambda.Code.fromAsset(config.codePath ?? '../solution/backend/', { + exclude: ['__pycache__', '*.pyc', '.git', '*.egg-info'] + }), memorySize: config.memorySize, timeout: cdk.Duration.seconds(config.timeout), role, From ff562c81595c5e70db0a86dacbd55ee0908c05da Mon Sep 17 00:00:00 2001 From: addis-samtek <155009976+addis-samtek@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:56:55 -0500 Subject: [PATCH 05/10] Add text extract stack cdk --- cdk-eregs/bin/cdk-eregs.ts | 46 +++++++++++++------------- cdk-eregs/lib/stacks/redirect-stack.ts | 5 ++- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/cdk-eregs/bin/cdk-eregs.ts b/cdk-eregs/bin/cdk-eregs.ts index 8096c4007..77772337a 100644 --- a/cdk-eregs/bin/cdk-eregs.ts +++ b/cdk-eregs/bin/cdk-eregs.ts @@ -83,29 +83,29 @@ async function main() { cdk.Tags.of(app).add(key, value); }); - // Create RedirectApiStack - const redirectStack = new RedirectApiStack(app, stageConfig.getResourceName('redirect-api'), { - lambdaConfig: { - runtime: lambda.Runtime.PYTHON_3_12, - memorySize: 1024, - timeout: 30, - }, - apiConfig: { - loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, - }, - }, stageConfig); - const maintenanceStack = new MaintenanceApiStack(app, stageConfig.getResourceName('maintenance-api'), { - lambdaConfig: { - runtime: lambda.Runtime.PYTHON_3_12, - memorySize: 1024, - timeout: 30, - }, - apiConfig: { - loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, - }, - }, stageConfig); - cdk.Tags.of(redirectStack).add('lambda-type', 'zip'); - cdk.Tags.of(maintenanceStack).add('lambda-type', 'zip'); + // // Create RedirectApiStack + // const redirectStack = new RedirectApiStack(app, stageConfig.getResourceName('redirect-api'), { + // lambdaConfig: { + // runtime: lambda.Runtime.PYTHON_3_12, + // memorySize: 1024, + // timeout: 30, + // }, + // apiConfig: { + // loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, + // }, + // }, stageConfig); + // const maintenanceStack = new MaintenanceApiStack(app, stageConfig.getResourceName('maintenance-api'), { + // lambdaConfig: { + // runtime: lambda.Runtime.PYTHON_3_12, + // memorySize: 1024, + // timeout: 30, + // }, + // apiConfig: { + // loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, + // }, + // }, stageConfig); + // cdk.Tags.of(redirectStack).add('lambda-type', 'zip'); + // cdk.Tags.of(maintenanceStack).add('lambda-type', 'zip'); const textExtractorStack = new TextExtractorStack(app, stageConfig.getResourceName('text-extractor'), { env, lambdaConfig: { diff --git a/cdk-eregs/lib/stacks/redirect-stack.ts b/cdk-eregs/lib/stacks/redirect-stack.ts index 96f44c199..bc61e51a6 100644 --- a/cdk-eregs/lib/stacks/redirect-stack.ts +++ b/cdk-eregs/lib/stacks/redirect-stack.ts @@ -109,9 +109,8 @@ export class RedirectApiStack extends cdk.Stack { description: `Redirect API Lambda function for ${this.stageConfig.environment} stage`, runtime: config.runtime, handler: config.handler ?? 'redirect_lambda.handler', - code: lambda.Code.fromAsset(config.codePath ?? '../solution/backend/', { - exclude: ['__pycache__', '*.pyc', '.git', '*.egg-info'] - }), + code: lambda.Code.fromAsset(config.codePath ?? '../solution/backend/'), + memorySize: config.memorySize, timeout: cdk.Duration.seconds(config.timeout), role, From a347959765c625ba54192126dedf41babbeec4cb Mon Sep 17 00:00:00 2001 From: addis-samtek <155009976+addis-samtek@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:24:38 -0500 Subject: [PATCH 06/10] Add text extract stack cdk --- cdk-eregs/lib/stacks/text-extract-stack.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdk-eregs/lib/stacks/text-extract-stack.ts b/cdk-eregs/lib/stacks/text-extract-stack.ts index b48951549..2255e8c8f 100644 --- a/cdk-eregs/lib/stacks/text-extract-stack.ts +++ b/cdk-eregs/lib/stacks/text-extract-stack.ts @@ -175,13 +175,13 @@ export class TextExtractorStack extends cdk.Stack { vpc: ec2.IVpc, securityGroup: ec2.SecurityGroup, ): lambda.Function { - const dockerContextPath = path.resolve(__dirname, '../../../solution/text-extractor/'); + const dockerContextPath = path.resolve(__dirname, '../../../solution/'); console.log('Docker context path:', dockerContextPath); return new lambda.DockerImageFunction(this, 'TextExtractorFunction', { functionName: this.stageConfig.getResourceName('text-extractor'), code: lambda.DockerImageCode.fromImageAsset(dockerContextPath, { - file: 'Dockerfile', + file: 'text-extractor/Dockerfile', }), vpc, securityGroups: [securityGroup], From 983d8dbf8697db7a4e57e02bb26789a5edcbfc5b Mon Sep 17 00:00:00 2001 From: addis-samtek <155009976+addis-samtek@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:27:15 -0500 Subject: [PATCH 07/10] Add text extract stack cdk --- cdk-eregs/bin/cdk-eregs.ts | 46 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/cdk-eregs/bin/cdk-eregs.ts b/cdk-eregs/bin/cdk-eregs.ts index 77772337a..8096c4007 100644 --- a/cdk-eregs/bin/cdk-eregs.ts +++ b/cdk-eregs/bin/cdk-eregs.ts @@ -83,29 +83,29 @@ async function main() { cdk.Tags.of(app).add(key, value); }); - // // Create RedirectApiStack - // const redirectStack = new RedirectApiStack(app, stageConfig.getResourceName('redirect-api'), { - // lambdaConfig: { - // runtime: lambda.Runtime.PYTHON_3_12, - // memorySize: 1024, - // timeout: 30, - // }, - // apiConfig: { - // loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, - // }, - // }, stageConfig); - // const maintenanceStack = new MaintenanceApiStack(app, stageConfig.getResourceName('maintenance-api'), { - // lambdaConfig: { - // runtime: lambda.Runtime.PYTHON_3_12, - // memorySize: 1024, - // timeout: 30, - // }, - // apiConfig: { - // loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, - // }, - // }, stageConfig); - // cdk.Tags.of(redirectStack).add('lambda-type', 'zip'); - // cdk.Tags.of(maintenanceStack).add('lambda-type', 'zip'); + // Create RedirectApiStack + const redirectStack = new RedirectApiStack(app, stageConfig.getResourceName('redirect-api'), { + lambdaConfig: { + runtime: lambda.Runtime.PYTHON_3_12, + memorySize: 1024, + timeout: 30, + }, + apiConfig: { + loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, + }, + }, stageConfig); + const maintenanceStack = new MaintenanceApiStack(app, stageConfig.getResourceName('maintenance-api'), { + lambdaConfig: { + runtime: lambda.Runtime.PYTHON_3_12, + memorySize: 1024, + timeout: 30, + }, + apiConfig: { + loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, + }, + }, stageConfig); + cdk.Tags.of(redirectStack).add('lambda-type', 'zip'); + cdk.Tags.of(maintenanceStack).add('lambda-type', 'zip'); const textExtractorStack = new TextExtractorStack(app, stageConfig.getResourceName('text-extractor'), { env, lambdaConfig: { From 264b8648ca9e7c8c25b463d180257056f543866b Mon Sep 17 00:00:00 2001 From: addis-samtek <155009976+addis-samtek@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:48:22 -0500 Subject: [PATCH 08/10] Add text extract stack cdk --- cdk-eregs/bin/cdk-eregs.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cdk-eregs/bin/cdk-eregs.ts b/cdk-eregs/bin/cdk-eregs.ts index 8096c4007..5fbd3959a 100644 --- a/cdk-eregs/bin/cdk-eregs.ts +++ b/cdk-eregs/bin/cdk-eregs.ts @@ -104,8 +104,7 @@ async function main() { loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, }, }, stageConfig); - cdk.Tags.of(redirectStack).add('lambda-type', 'zip'); - cdk.Tags.of(maintenanceStack).add('lambda-type', 'zip'); + const textExtractorStack = new TextExtractorStack(app, stageConfig.getResourceName('text-extractor'), { env, lambdaConfig: { @@ -120,8 +119,6 @@ async function main() { httpPassword, } }, stageConfig); - cdk.Tags.of(textExtractorStack).add('lambda-type', 'docker'); - // Apply aspects await applyGlobalAspects(app, stageConfig); From 150041a4619a04c4385da7001c2388849b5455b5 Mon Sep 17 00:00:00 2001 From: addis-samtek <155009976+addis-samtek@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:54:17 -0500 Subject: [PATCH 09/10] Add text extract stack cdk --- .github/workflows/deploy-experimental.yml | 6 +++--- .github/workflows/remove-experimental.yml | 22 ++++++++++++++++++++++ cdk-eregs/bin/cdk-eregs.ts | 6 +++--- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-experimental.yml b/.github/workflows/deploy-experimental.yml index 378afc063..f6745e1a2 100644 --- a/.github/workflows/deploy-experimental.yml +++ b/.github/workflows/deploy-experimental.yml @@ -156,7 +156,7 @@ jobs: npm install cdk deploy "*redirect-api" \ -c environment=${{ env.ENVIRONMENT_NAME }} \ - --require-approval never --tags lambda-type=zip + --require-approval never popd deploy-Maintenance-Api-CDK: environment: @@ -199,7 +199,7 @@ jobs: npm install cdk deploy "*maintenance-api" \ -c environment=${{ env.ENVIRONMENT_NAME }} \ - --require-approval never --tags lambda-type=zip + --require-approval never popd deploy-Text-extract-CDK: environment: @@ -242,7 +242,7 @@ jobs: npm install cdk deploy "*-text-extractor" \ -c environment=${{ env.ENVIRONMENT_NAME }} \ - --require-approval never --tags lambda-type=docker + --require-approval never popd deploy-text-extractor: environment: diff --git a/.github/workflows/remove-experimental.yml b/.github/workflows/remove-experimental.yml index e2178d3bf..aa8e6821a 100644 --- a/.github/workflows/remove-experimental.yml +++ b/.github/workflows/remove-experimental.yml @@ -145,5 +145,27 @@ jobs: -c environment=${{ env.ENVIRONMENT_NAME }} \ --force + echo "Cleanup completed for stack: ${STACK_NAME}" + popd + - name: Destroy PR-Text-extract Stack + env: + PR_NUMBER: ${{ github.event.number }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + CDK_DEBUG: true + ENVIRONMENT_NAME: ${{ env.ENVIRONMENT_NAME }} + run: | + pushd cdk-eregs + npm install -g aws-cdk@latest @aws-sdk/client-ssm + npm install + + # Generate the exact stack name for this PR + STACK_NAME="cms-eregs-eph-${PR_NUMBER}-text-extractor" + + echo "Destroying PR-specific stack: ${STACK_NAME}" + cdk destroy "${STACK_NAME}" \ + -c environment=${{ env.ENVIRONMENT_NAME }} \ + --force + echo "Cleanup completed for stack: ${STACK_NAME}" popd \ No newline at end of file diff --git a/cdk-eregs/bin/cdk-eregs.ts b/cdk-eregs/bin/cdk-eregs.ts index 5fbd3959a..fbfd4f134 100644 --- a/cdk-eregs/bin/cdk-eregs.ts +++ b/cdk-eregs/bin/cdk-eregs.ts @@ -84,7 +84,7 @@ async function main() { }); // Create RedirectApiStack - const redirectStack = new RedirectApiStack(app, stageConfig.getResourceName('redirect-api'), { + new RedirectApiStack(app, stageConfig.getResourceName('redirect-api'), { lambdaConfig: { runtime: lambda.Runtime.PYTHON_3_12, memorySize: 1024, @@ -94,7 +94,7 @@ async function main() { loggingLevel: cdk.aws_apigateway.MethodLoggingLevel.INFO, }, }, stageConfig); - const maintenanceStack = new MaintenanceApiStack(app, stageConfig.getResourceName('maintenance-api'), { + new MaintenanceApiStack(app, stageConfig.getResourceName('maintenance-api'), { lambdaConfig: { runtime: lambda.Runtime.PYTHON_3_12, memorySize: 1024, @@ -105,7 +105,7 @@ async function main() { }, }, stageConfig); - const textExtractorStack = new TextExtractorStack(app, stageConfig.getResourceName('text-extractor'), { + new TextExtractorStack(app, stageConfig.getResourceName('text-extractor'), { env, lambdaConfig: { memorySize: 1024, From b7478210292dc4bac91f250f8b4d76e4a52b170f Mon Sep 17 00:00:00 2001 From: addis-samtek <155009976+addis-samtek@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:16:46 -0500 Subject: [PATCH 10/10] Add text extract stack cdk --- .github/workflows/deploy-experimental.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-experimental.yml b/.github/workflows/deploy-experimental.yml index f6745e1a2..cf13cf0a6 100644 --- a/.github/workflows/deploy-experimental.yml +++ b/.github/workflows/deploy-experimental.yml @@ -201,7 +201,7 @@ jobs: -c environment=${{ env.ENVIRONMENT_NAME }} \ --require-approval never popd - deploy-Text-extract-CDK: + deploy-Text-extractor-CDK: environment: name: "dev" env: @@ -227,7 +227,7 @@ jobs: role-to-assume: ${{ secrets.AWS_OIDC_ROLE_TO_ASSUME }} aws-region: us-east-1 - name: Deploy Text extract via CDK - id: deploy-text-extract + id: deploy-text-extractor if: success() && steps.findPr.outputs.number env: PR_NUMBER: ${{ steps.findPr.outputs.pr }}