From e0b07d4c39315078b64044dbd3da9c2702c0646b Mon Sep 17 00:00:00 2001 From: Caleb Godwin <75450124+cgodwin1@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:01:04 -0500 Subject: [PATCH] EREGCSC-2894 Update CDK bootstrap for latest version (#1488) * chore: add github action to update CDK bootstrap * fix: add action environment * fix: empty directory for cdk init * fix: replace default template role with eregs admin role * test: run bootstrap update on val too * feat: create script to update default CDK bootstrap template with CMS specifics * fix: remove template.yaml from repo * fix: remove template.yaml exception from gitignore * fix: typo * fix: update_template.py help message * feat: update script * fix: add silent option for curl * test: bootstrap on val and prod * feat: schedule cdk bootstrap update weekly * fix: replace the eregs-specific default role with macfcee * feat: implement final suggestions from reviewers * add README.md * add requirements.txt due to PyYAML dependency * make role-to-assume a required positional argument --- .github/workflows/update-cdk-bootstrap.yml | 67 +++++++++ cdk-eregs/bootstrap/.gitignore | 6 + cdk-eregs/bootstrap/README.md | 140 ++++++++++++++++++ cdk-eregs/bootstrap/requirements.txt | 1 + cdk-eregs/bootstrap/roles.json | 27 ++++ cdk-eregs/bootstrap/update_template.py | 159 +++++++++++++++++++++ 6 files changed, 400 insertions(+) create mode 100644 .github/workflows/update-cdk-bootstrap.yml create mode 100644 cdk-eregs/bootstrap/.gitignore create mode 100644 cdk-eregs/bootstrap/README.md create mode 100644 cdk-eregs/bootstrap/requirements.txt create mode 100644 cdk-eregs/bootstrap/roles.json create mode 100755 cdk-eregs/bootstrap/update_template.py diff --git a/.github/workflows/update-cdk-bootstrap.yml b/.github/workflows/update-cdk-bootstrap.yml new file mode 100644 index 000000000..47d5d54b9 --- /dev/null +++ b/.github/workflows/update-cdk-bootstrap.yml @@ -0,0 +1,67 @@ +name: Update CDK Bootstrap + +on: + schedule: + - cron: '0 0 * * 1' # Runs every Monday + +permissions: + id-token: write + contents: read + actions: read + +jobs: + update-cdk-bootstrap: + strategy: + max-parallel: 1 + matrix: + environment: ["dev", "val", "prod"] + + runs-on: ubuntu-latest + + environment: + name: ${{ matrix.environment }} + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Configure AWS credentials for GitHub Actions + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_TO_ASSUME }} + aws-region: us-east-1 + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: 18.14 + + - name: Install AWS CDK + run: npm install -g aws-cdk + + - name: Update CDK Bootstrap + env: + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + run: | + pushd cdk-eregs/bootstrap + + echo "Downloading latest bootstrap template..." + curl -s "https://raw.githubusercontent.com/aws/aws-cdk/refs/heads/main/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml" -o latest-template.yaml > /dev/null + + echo "Applying CMS-specific changes to the template..." + pip install -r requirements.txt + ./update_template.py roles.json latest-template.yaml template.yaml ct-ado-eregs-application-admin + + echo "Creating temporary CDK app for bootstrap update..." + mkdir temp; pushd temp + cdk init app --language=typescript > /dev/null + cp ../template.yaml . + + echo "Bootstrapping CDK environment..." + cdk bootstrap --template template.yaml \ + --cloudformation-execution-policies arn:aws:iam::aws:policy/AdministratorAccess,arn:aws:iam::${AWS_ACCOUNT_ID}:policy/ADO-Restriction-Policy,arn:aws:iam::${AWS_ACCOUNT_ID}:policy/CMSApprovedAWSServices \ + --custom-permissions-boundary ct-ado-poweruser-permissions-boundary-policy \ + --qualifier one + + popd; popd diff --git a/cdk-eregs/bootstrap/.gitignore b/cdk-eregs/bootstrap/.gitignore new file mode 100644 index 000000000..bdaab8694 --- /dev/null +++ b/cdk-eregs/bootstrap/.gitignore @@ -0,0 +1,6 @@ +* +!update_template.py +!roles.json +!.gitignore +!README.md +!requirements.txt diff --git a/cdk-eregs/bootstrap/README.md b/cdk-eregs/bootstrap/README.md new file mode 100644 index 000000000..6b5349322 --- /dev/null +++ b/cdk-eregs/bootstrap/README.md @@ -0,0 +1,140 @@ +# Update Template Script + +## Overview + +In order for our CDK scripts to reliably deploy/create AWS resources, we need to keep our CDK stacks up to date. + +We decided we wanted to do this automatically via cron'd Github Action, but found it wasn't possible to do this using standard bootstrap procedure because we are relying on a CMS-specific `template.yaml`. By diffing the CMS `template.yaml` with the [default one from CDK](https://github.com/aws/aws-cdk/blob/main/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml), we found it was feasible to create a script (`update_template.py`) that attempts to automatically apply those needed changes to the default template. + +From there, we can automate the bootstrap update procedure. + +## Prerequisites + +- Python 3.x +- Required Python packages (listed in `requirements.txt`) +- A copy of the [default template.yaml from CDK](https://github.com/aws/aws-cdk/blob/main/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml) +- A role to assume (for eRegs it is `ct-ado-eregs-application-admin`, for example) + +## Setup + +First, download the default `template.yaml`. Then, create a file called `roles.json`. This file contains a list of dictionaries like so: + +```jsonc +[ + { + "name": "FilePublishingRole", + "nested_policy": false, + "update_policy": true + }, + ....... more roles ....... +] +``` + +Where `name` is the name of the role to update, `nested_policy` is a boolean indicating whether the policy document to update is nested in an Fn::If block, and `update_policy` specifies whether to attempt to update the policy document at all. + +Note that if a role is specified in the list, regardless of the status of the `update_policy` boolean, the script will _always_ add a role path and permissions boundary. This boolean only tells the script if it should _also_ update the `AssumeRolePolicyDocument` block of the YAML for that role. + +## Usage + +To run the `update_template.py` script, use the following command: +```sh +./update_template.py +``` +Where `role file` is the filename of the `roles.json` you created earlier, `input template` is the default CDK template.yaml file, `output filename` is the filename to write the updated template to, and `name of role to assume` is the role unique to your project. + +## Additional options + +In addition to the four positional arguments that `update_template.yaml` takes, you can also specify the following: + +`--boundary-policy-arn-prefix BOUNDARY_POLICY_ARN_PREFIX`
+ARN prefix of the permissions boundary policy to attach to the roles. Default is: `arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/`. + +`--boundary-policy-name BOUNDARY_POLICY_NAME`
+Name of the permissions boundary policy to attach to the roles. Default is: `cms-cloud-admin/ct-ado-poweruser-permissions-boundary-policy`. + +`--role-to-assume-arn-prefix ROLE_TO_ASSUME_ARN_PREFIX`
+ARN prefix of the role to be added to the AssumeRolePolicyDocument. Default is: `arn:aws:iam::${AWS::AccountId}:role/`. + +`--role-path ROLE_PATH`
+Path to be added to the role properties. Default is: `/delegatedadmin/developer/`. + +## Automating CDK updates via Github Actions + +This sample script should provide the starting point for a script that can automatically update the CDK bootstrap: + +```yaml +name: Update CDK Bootstrap + +on: + schedule: + - cron: '0 0 * * 1' # Runs every Monday + +permissions: + id-token: write + contents: read + actions: read + +jobs: + update-cdk-bootstrap: + strategy: + max-parallel: 1 + matrix: + environment: ["dev", "val", "prod"] + + runs-on: ubuntu-latest + + environment: + name: ${{ matrix.environment }} + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Configure AWS credentials for GitHub Actions + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_TO_ASSUME }} + aws-region: us-east-1 + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: 18.14 + + - name: Install AWS CDK + run: npm install -g aws-cdk # Install CDK + + - name: Update CDK Bootstrap + env: + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} # Get the account ID and region + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + run: | + pushd cdk-eregs/bootstrap + + # Use curl to download the latest template.yaml directly from the CDK repo + curl -s "https://raw.githubusercontent.com/aws/aws-cdk/refs/heads/main/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml" -o latest-template.yaml > /dev/null + + # Install script requirements + pip install -r requirements.txt + + # Use update_template.py to generate our custom template.yaml + # Note the role-to-assume name of 'ct-ado-eregs-application-admin', + # update this for your use-case. + ./update_template.py roles.json latest-template.yaml template.yaml ct-ado-eregs-application-admin + + # Create a temporary CDK app for bootstrapping purposes only + mkdir temp; pushd temp + cdk init app --language=typescript > /dev/null + + # Copy the default template into the new CDK app + cp ../template.yaml . + + # Run cdk bootstrap with the custom template + # Note other account-specific parameters that can be adjusted for your use-case + cdk bootstrap --template template.yaml \ + --cloudformation-execution-policies arn:aws:iam::aws:policy/AdministratorAccess,arn:aws:iam::${AWS_ACCOUNT_ID}:policy/ADO-Restriction-Policy,arn:aws:iam::${AWS_ACCOUNT_ID}:policy/CMSApprovedAWSServices \ + --custom-permissions-boundary ct-ado-poweruser-permissions-boundary-policy \ + --qualifier one + + popd; popd +``` diff --git a/cdk-eregs/bootstrap/requirements.txt b/cdk-eregs/bootstrap/requirements.txt new file mode 100644 index 000000000..5500f007d --- /dev/null +++ b/cdk-eregs/bootstrap/requirements.txt @@ -0,0 +1 @@ +PyYAML diff --git a/cdk-eregs/bootstrap/roles.json b/cdk-eregs/bootstrap/roles.json new file mode 100644 index 000000000..f091f7b0c --- /dev/null +++ b/cdk-eregs/bootstrap/roles.json @@ -0,0 +1,27 @@ +[ + { + "name": "FilePublishingRole", + "nested_policy": false, + "update_policy": true + }, + { + "name": "DeploymentActionRole", + "nested_policy": false, + "update_policy": true + }, + { + "name": "LookupRole", + "nested_policy": false, + "update_policy": true + }, + { + "name": "ImagePublishingRole", + "nested_policy": true, + "update_policy": true + }, + { + "name": "CloudFormationExecutionRole", + "nested_policy": false, + "update_policy": false + } +] diff --git a/cdk-eregs/bootstrap/update_template.py b/cdk-eregs/bootstrap/update_template.py new file mode 100755 index 000000000..7d9968384 --- /dev/null +++ b/cdk-eregs/bootstrap/update_template.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +import sys +import yaml +import json +import argparse + + +# Default values for the permissions boundary policy prefix & name, role to assume prefix, and role path +# You can use CLI flags to change these values; run the script with "-h" for more information +BOUNDARY_POLICY_ARN_PREFIX = "arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/" +BOUNDARY_POLICY_NAME = "cms-cloud-admin/ct-ado-poweruser-permissions-boundary-policy" +ROLE_TO_ASSUME_ARN_PREFIX = "arn:aws:iam::${AWS::AccountId}:role/" +ROLE_TO_ASSUME_NAME = None +ROLE_PATH = "/delegatedadmin/developer/" + + +# Update the role properties to add the correct path and permissions boundary +def update_role_properties(properties: dict): + properties["Path"] = ROLE_PATH + properties["PermissionsBoundary"] = { + "Fn::Sub": BOUNDARY_POLICY_ARN_PREFIX + BOUNDARY_POLICY_NAME, + } + return properties + + +# Check if the statement matches the expected format for adding the role +def statement_matches(statement: dict): + return all([ + statement.get("Action") == "sts:AssumeRole", + statement.get("Effect") == "Allow", + "Principal" in statement, + type(statement["Principal"]) is dict, + "AWS" in statement["Principal"], + ]) + + +# Update the principal to add the correct role +def update_principal(principal: dict): + return {"AWS": [ + principal["AWS"], + {"Fn::Sub": ROLE_TO_ASSUME_ARN_PREFIX + ROLE_TO_ASSUME_NAME}, + ]} + + +# Update the policy document statements to add the correct role as a principal in the correct place +def update_policy_doc_statements(yaml_statements: type[list | dict], nested: bool): + statements = [yaml_statements] if type(yaml_statements) is not list else yaml_statements + for i in statements: + if nested and i.get("Fn::If"): + ifs = [i["Fn::If"]] if type(i["Fn::If"]) is not list else i["Fn::If"] + for j in [statement for statement in ifs if type(statement) is dict]: + if statement_matches(j): + j["Principal"] = update_principal(j["Principal"]) + return statements + elif not nested and statement_matches(i): + i["Principal"] = update_principal(i["Principal"]) + return statements + return yaml_statements + + +# Process the role data to update the role properties and policy document statements +def process_role(role: dict, data: dict): + try: + data["Properties"] = update_role_properties(data["Properties"]) + except Exception as e: + print(f'Error updating role properties for {role["name"]}: {e}') + sys.exit(1) + + if role["update_policy"]: + try: + data["Properties"]["AssumeRolePolicyDocument"]["Statement"] = update_policy_doc_statements( + data["Properties"]["AssumeRolePolicyDocument"]["Statement"], + role["nested_policy"], + ) + except Exception as e: + print(f'Error updating policy document statements for {role["name"]}: {e}') + sys.exit(1) + + return data + + +# Custom YAML dumper to ensure that the output is indented correctly +class IndentedDumper(yaml.Dumper): + def increase_indent(self, flow: bool = False, indentless: bool = False): + return super().increase_indent(flow, False) + + +if __name__ == '__main__': + # Parse command line arguments + parser = argparse.ArgumentParser( + description='Utility to update the default CDK bootstrapping YAML file with CMS-specific role properties.', + epilog='Note that ANY errors during processing will result in the script exiting with a non-zero status code. ' + 'This is intentional to prevent a bad bootstrapping attempt if the template is not updated correctly.', + ) + + parser.add_argument( + 'role_file', type=str, + help='JSON file containing the role information as a list of dictionaries. Each entry must contain the following ' + 'keys: "name" is the name of the role, "update_policy" is a bool indicating whether to update the policy ' + 'document, and "nested_policy" is a bool indicating whether the document to update is nested in an Fn::If block.', + ) + parser.add_argument('input_template', type=str, help='YAML file containing the default CDK bootstrapping template.') + parser.add_argument('output_template', type=str, help='YAML file to write the updated CDK bootstrapping template to.') + parser.add_argument('role_to_assume_name', type=str, help='Name of the role to be added to the AssumeRolePolicyDocument.') + parser.add_argument('--boundary-policy-arn-prefix', type=str, default=BOUNDARY_POLICY_ARN_PREFIX, + help='ARN prefix of the permissions boundary policy to attach to the roles. ' + f'Default: "{BOUNDARY_POLICY_ARN_PREFIX}".') + parser.add_argument('--boundary-policy-name', type=str, default=BOUNDARY_POLICY_NAME, + help='Name of the permissions boundary policy to attach to the roles. ' + f'Default: "{BOUNDARY_POLICY_NAME}".') + parser.add_argument('--role-to-assume-arn-prefix', type=str, default=ROLE_TO_ASSUME_ARN_PREFIX, + help='ARN prefix of the role to be added to the AssumeRolePolicyDocument. ' + f'Default: "{ROLE_TO_ASSUME_ARN_PREFIX}".') + parser.add_argument('--role-path', type=str, default=ROLE_PATH, + help=f'Path to be added to the role properties. Default: "{ROLE_PATH}".') + args = parser.parse_args() + + BOUNDARY_POLICY_ARN_PREFIX = args.boundary_policy_arn_prefix + BOUNDARY_POLICY_NAME = args.boundary_policy_name + ROLE_TO_ASSUME_ARN_PREFIX = args.role_to_assume_arn_prefix + ROLE_TO_ASSUME_NAME = args.role_to_assume_name + ROLE_PATH = args.role_path + + # Read the file containing role information + try: + with open(args.role_file, 'r') as f: + roles = json.load(f) + except FileNotFoundError: + print(f'Role file {args.role_file} not found') + sys.exit(1) + + # Read the input CDK bootstrap template YAML file + try: + with open(args.input_template, 'r') as f: + data = yaml.safe_load(f) + except FileNotFoundError: + print(f'Input template file {args.input_template} not found') + sys.exit(1) + + # Process each role in the role file + for role in roles: + if not all(["name" in role, "update_policy" in role, "nested_policy" in role]): + print('Role information must contain "name", "update_policy", and "nested_policy" keys') + sys.exit(1) + name = role["name"] + + if name not in data["Resources"]: + print(f'Role "{name}" not found in template') + sys.exit(1) + + data["Resources"][name] = process_role(role, data["Resources"][name]) + + # Write the updated template to the output file + with open(args.output_template, 'w') as f: + yaml.dump(data, f, default_flow_style=False, sort_keys=False, Dumper=IndentedDumper) + + print(f'Updated template written to {args.output_template}') + sys.exit(0)