Merge pull request #104 from zxkane/pipeline

ci/cd pipeline support

.github/workflows/pipeline.yml

@@ -0,0 +1,22 @@
+name: CD pipeline
+
+on:
+  # Trigger the workflow on push for the master branch
+  push:
+    branches:
+      - master
+
+jobs:
+  trigger-job:
+    name: Pipeline Trigger
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Trigger pipeline on AWS
+      id: trigger-pipeline
+      env:
+        COMMIT: ${{ github.sha }}
+        TRIGGER_URL: ${{ secrets.PIPELINE_TRIGGER_URL }}
+      run: |
+        # Trigger pipeline state machine
+        if [ $(curl -LI -s -o /dev/null -w '%{http_code}\n' -X PUT $TRIGGER_URL?commit=$COMMIT) != "200" ]; then exit 1; fi

.github/workflows/test.yml

@@ -25,7 +25,7 @@ jobs:
       uses: actions/setup-node@v1
       with:
         node-version: ${{ matrix.node-version }}
-    - run: npm install
+    - run: npm run install-deps
     - run: npm run build --if-present
     - run: npm test
       env:

README.md

@@ -47,6 +47,7 @@ It consits of below independent [stacks][cfn-stack],
   - Monitor stack
     - create several CodeBuild projects to verify integrity of index files
     - run projects periodically and report upon failure
+- [Pipeline stack](pipeline.md)
 
 ## Prerequisites
 - VPC with both public and private subnets crossing two AZs at least and NAT gateway. You can [deploy the network stack](#deploy-network-stackoptional) if you don't have a VPC sastfied the requirements.
@@ -96,7 +97,7 @@ npx cdk deploy OpenTunaStack -c vpcId=<existing vpc Id>
 npx cdk deploy OpenTunaStack -c vpcId=<existing vpc Id> -c slackHookUrl=<webhook url>
 
 # or deploy with existing EFS filesystem
-npx cdk deploy OpenTunaStack -c vpcId=<existing vpc Id> -c fileSystemId=<existing filesystem id>
+npx cdk deploy OpenTunaStack -c vpcId=<existing vpc Id> -c fileSystemId=<existing filesystem id> -c fileSystemSGId=<existing sg id of the given file system>
 
 # deploy with domain name and use Route53 as DNS resolver
 npx cdk deploy OpenTunaStack -c vpcId=<existing vpc Id> -c domainName=<domain name of site> -c domainZone=<public hosted zone of your domain in Route53>

bin/opentuna-pipeline-setup.ts

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+import 'source-map-support/register';
+import * as cdk from '@aws-cdk/core';
+import * as iam from '@aws-cdk/aws-iam';
+
+const app = new cdk.App();
+
+const appPrefix = app.node.tryGetContext('stackPrefix') || 'OpenTuna';
+const env = {
+    region: process.env.CDK_DEFAULT_REGION,
+    account: process.env.CDK_DEFAULT_ACCOUNT,
+};
+
+const suffix = app.node.tryGetContext('stackSuffix') || '';
+
+const trustedAccount = app.node.tryGetContext('trustedAccount');
+if (!trustedAccount) {
+    throw new Error(`Pls specify the trusted account for pipeline deployment via context 'trustedAccount'.`);
+}
+
+const stack = new cdk.Stack(app, `PipelineCrossAccountDeploymentSetupStack`, {
+    env,
+});
+
+// the role to assume when the CDK is in write mode, i.e. deploy
+// allow roles from the trusted account to assume this role
+const openTunaDeployRole = new iam.Role(stack, 'DeployRole', {
+    assumedBy: new iam.AccountPrincipal(trustedAccount),
+    roleName: `opentuna-deployment-trust-${trustedAccount}-role`,
+});
+
+// Attach the AdministratorAccess policy to this role.
+openTunaDeployRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('AdministratorAccess'));
+
+cdk.Tags.of(app).add('app', `${appPrefix}${suffix}`);
+
+new cdk.CfnOutput(stack, `DeployRoleFor${trustedAccount}`, {
+    value: `${openTunaDeployRole.roleArn}`,
+    description: `Deployment role for trusted account ${trustedAccount}.`
+});

bin/opentuna-pipeline.ts

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+import 'source-map-support/register';
+import * as cdk from '@aws-cdk/core';
+import * as fs from 'fs';
+import * as path from 'path';
+import { PipelineStack, Stage } from '../lib/pipeline-stack';
+import { CommonStack } from '../lib/common-stack';
+
+const app = new cdk.App();
+
+const appPrefix = app.node.tryGetContext('stackPrefix') || 'OpenTuna';
+const env = { 
+    region: process.env.CDK_DEFAULT_REGION,
+    account: process.env.CDK_DEFAULT_ACCOUNT,
+};
+
+const suffix = app.node.tryGetContext('stackSuffix') || '';
+
+const commonStack = new CommonStack(app, `${appPrefix}CommonStack${suffix}`, {
+    env,
+});
+
+// workaround for CDK StringParameter.valueFromLookup
+// see https://github.com/aws/aws-cdk/issues/8699 for detail
+const uatJsonFile = app.node.tryGetContext('UATConf') || `../cdk.out/uat.json`;
+const uat: Stage = JSON.parse(fs.readFileSync(path.join(__dirname, uatJsonFile), 'utf-8'));
+const prodJsonFile = app.node.tryGetContext('ProdConf') || `../cdk.out/prod.json`;
+const prod: Stage = JSON.parse(fs.readFileSync(path.join(__dirname, prodJsonFile), 'utf-8'));
+
+new PipelineStack(app, `${appPrefix}PipelineStack${suffix}`, {
+    env,
+    topic: commonStack.notifyTopic,
+    uat,
+    prod,
+});
+
+cdk.Tags.of(app).add('app', `${appPrefix}${suffix}`);

bin/opentuna.ts

@@ -10,7 +10,7 @@ const app = new cdk.App();
 const appPrefix = app.node.tryGetContext('stackPrefix') || 'OpenTuna';
 const env = { 
     region: process.env.CDK_DEFAULT_REGION,
-    account: process.env.CDK_DEFAULT_ACCOUNT,
+    account: app.node.tryGetContext('accountId') ?? process.env.CDK_DEFAULT_ACCOUNT,
 };
 
 const suffix = app.node.tryGetContext('stackSuffix') || '';

lib/common-stack.ts

@@ -44,6 +44,7 @@ export class CommonStack extends cdk.Stack {
                 code: lambda.Code.fromAsset(path.join(__dirname, './lambda.d/slack-webhook')),
                 environment: {
                     SLACK_WEBHOOK_URL: slackHookUrl,
+                    CHANNEL: this.node.tryGetContext('slackChannel'),
                 },
             });
             this.notifyTopic.addSubscription(new sns_sub.LambdaSubscription(slackSubscription));

lib/lambda.d/slack-webhook/index.py

@@ -53,6 +53,37 @@ def handler(event, context):
                         Check build {message.get('sanityBuildId')} for detail info."""),
                 "icon_emoji": icon_emoji,
             }
+        elif msgType == 'pipeline':
+            state = message.get("state")
+            if state == 'approval':
+                slackMsg = {
+                    "channel": channel,
+                    "username": 'Pipeline',
+
+                    "text": textwrap.dedent(f"""\
+                            OpenTUNA pipeline '{message.get('stateMachineName')}' is going to next stage '{message.get('nextStage')}' on commit '{message.get('commit')}'.
+
+                            Check stage '{message.get('stage')}' via https://{message.get('domain')},
+
+                            NOTE: below actions are one-time actions and the operation can NOT be revoked. 
+                            Also below actions will be expired in {message.get('timeout')} minutes.
+
+                            <{message.get('approveAction')}|Click to Approve>
+                            
+                            <{message.get('rejectAction')}|Click to Reject>
+                            """),
+                    "icon_emoji": ":question:",
+                }
+            else:
+                buildRT = message.get('result')
+                slackMsg = {
+                    "channel": channel,
+                    "username": 'Pipeline',
+                    "text": textwrap.dedent(f"""\
+                            OpenTUNA pipeline stage '{message.get('stage')}' on commit '{message.get('commit')}' is {buildRT}.,
+                            """),
+                    "icon_emoji": ":thumbsup:" if buildRT == "succeeded" else ":thumbsdown:",
+                }
         else:
             print(f"Ignore non-alarm message.")
 

lib/lambda.pipelines.d/approver-actions/index.ts

@@ -0,0 +1,55 @@
+import { APIGatewayProxyHandlerV2 } from 'aws-lambda';
+import * as aws from 'aws-sdk';
+
+const stepfunctions = new aws.StepFunctions();
+
+export const pipelineApprovalAction: APIGatewayProxyHandlerV2 = async (event, _context, callback) => {
+    console.info(`Receiving pipeline approval action event ${JSON.stringify(event, null, 2)}.`);
+
+    // workaround to disable slack link auto preview
+    // https://slack.com/help/articles/204399343-Share-links-and-set-preview-preferences
+    if (event.requestContext.http.userAgent.indexOf('Slackbot') > -1) {
+        return {
+            statusCode: 401,
+            body: `This url does not support Slack LinkExpanding.`,
+        };
+    }
+
+    var message = {};
+
+    const action = event.queryStringParameters?.action;
+
+    if (action === "approve") {
+        message = { "Status": "Approved" };
+    } else if (action === "reject") {
+        message = { "Status": "Rejected" };
+    } else {
+        console.error(`Unrecognized action "${action}". Expected: approve, reject.`);
+        return {
+            statusCode: 400,
+            body: `Failed to process the request. Unrecognized Action "${action}".`,
+        };
+    }
+
+    const taskToken = event.queryStringParameters!.taskToken;
+    const statemachineName = event.queryStringParameters!.sm;
+    const executionName = event.queryStringParameters!.ex;
+
+    try {
+        await stepfunctions.sendTaskSuccess({
+            output: JSON.stringify(message),
+            taskToken: taskToken,
+        }).promise();
+    } catch (err) {
+        console.error(err, err.stack);
+        return {
+            statusCode: 500,
+            body: err.message,
+        }
+    }
+
+    return { 
+        statusCode: 200,
+        body: `Deployment pipeline "${statemachineName}" with execution "${executionName}" is ${action === 'approve' ? 'approved' : 'rejected'}.`,
+    };
+}