Merge branch 'main' into feature/deploy-pipeline-infra

CHANGELOG.md

@@ -14,9 +14,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - added `ray-on-eks`, and `manifests/ray-on-eks` manifests
 - added a `sagemaker-model-monitoring-module` module with an example of data quality, model quality, model bias, and model explainability monitoring of a SageMaker Endpoint
 - added an option to enable data capture in the `sagemaker-endpoint-module`
-- added a `personas` example module to deploy various roles required for an AI/ML project 
+- added a `personas` example module to deploy various roles required for an AI/ML project
+- added `sagemaker-model-cicd` module
 
 ### **Changed**
+
 - remove explicit module manifest account/region mappings from `fmops-qna-rag`
 - moved CI/CD infra to separate repository and added self mutation pipeline to provision infra for module `sagemaker-templates-service-catalog`
 - changed ECR encryption to KMS_MANAGED
@@ -33,14 +35,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## v1.2.0
 
 ### **Added**
+
 - added multi-acc sagemaker-mlops manifest example
 
 ### **Changed**
+
 - fixed model deploy cross-account permissions
 - added bucket and model package group names as stack outputs in the `sagemaker-templates` module
 - refactor inputs for the following modules to use Pydantic:
   - `mlflow-fargate`
-  - `mlflow-image` 
+  - `mlflow-image`
   - `sagemaker-studio`
   - `sagemaker-endpoint`
   - `sagemaker-templates-service-catalog`

data/mwaa/requirements/constraint-requirements.txt

@@ -231,7 +231,7 @@ cachetools==5.3.2
 cassandra-driver==3.29.0
 cattrs==23.2.3
 celery==5.3.6
-certifi==2023.11.17
+certifi==2024.7.4
 cffi==1.16.0
 cfgv==3.4.0
 cfn-lint==0.83.8

manifests/mlops-sagemaker-multiacc/deployment.yaml

@@ -12,7 +12,16 @@ groups:
     path: manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml
   - name: sagemaker-kernels
     path: manifests/mlops-sagemaker-multiacc/kernels-modules.yaml
+  - name: model-cicd
+    path: manifests/mlops-sagemaker-multiacc/sagemaker-model-cicd-modules.yaml
 targetAccountMappings:
+  - alias: tooling
+    accountId:
+      valueFrom:
+        envVariable: PRIMARY_ACCOUNT
+    regionMappings:
+      - region: us-east-1
+        default: true
   - alias: dev
     accountId:
       valueFrom:

manifests/mlops-sagemaker-multiacc/sagemaker-model-cicd-modules.yaml

@@ -0,0 +1,21 @@
+name: model # replace with name of the ML model you prefer
+path: modules/sagemaker/sagemaker-model-cicd
+targetAccount: tooling
+parameters:
+  - name: infra-repo
+    value:
+      host: codecommit
+      name: mlops-infra-repo
+  - name: model-build-repo
+    value:
+      host: codecommit
+      name: mlops-build-repo
+  - name: deployment-groups
+    value:
+      - name: experimentation
+        sourceBranch: dev
+        buildEnvironment:
+          name: dev
+          account: "REPLACE_WITH_DEV_ACCOUNT"
+          region: us-east-1
+          type: dev

modules/sagemaker/sagemaker-model-cicd/.eslintrc

@@ -0,0 +1,33 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "env": {
+    "node": true
+  },
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended"
+  ],
+  "parserOptions": {
+    "sourceType": "module"
+  },
+  "rules": {
+    "no-unused-vars": "off",
+    "@typescript-eslint/no-unused-vars": [
+      "error",
+      {
+        "args": "none"
+      }
+    ],
+    "@typescript-eslint/ban-ts-comment": "off",
+    "no-prototype-builtins": "off",
+    "@typescript-eslint/no-empty-function": "off"
+  },
+  "ignorePatterns": [
+    "cdk.out/"
+  ]
+}

modules/sagemaker/sagemaker-model-cicd/.gitignore

@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

modules/sagemaker/sagemaker-model-cicd/.npmignore

@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

modules/sagemaker/sagemaker-model-cicd/.prettierrc.json

@@ -0,0 +1,5 @@
+{
+  "singleQuote": true,
+  "overrides": [],
+  "plugins": ["prettier-plugin-organize-imports"]
+}

modules/sagemaker/sagemaker-model-cicd/README.md

@@ -0,0 +1,80 @@
+# SageMaker Model CICD
+
+## Description
+
+This module automates the creation of a comprehensive CICD pipeline using AWS CodePipelines, specifically designed for training and deploying SageMaker models. It embraces the Git Flow methodology, enabling model training on various datasets corresponding to different branches. For instance, models trained on the `dev` branch can be deployed to development environments, while those trained on the `main` branch are deployed to production environments.
+
+## Architecture
+
+![MLOPs](docs/_static/MLOps_Architecture.png)
+
+## Deployment
+
+```bash
+seedfarmer apply manifests/mlops-sagemaker-multiacc/deployment.yaml
+```
+
+## Inputs/Outputs
+
+### Input Parameters
+
+#### Required
+
+- `infra-repo`: The repository containing the CDK infrastructure code.
+- `model-build-repo`: The repository containing the model training code.
+- `deployment-groups`: List of deployment groups with source branch, build environment, and deploy environments.
+
+#### Optional
+
+#### Input Example
+
+```yaml
+name: sample-ml-model-name
+path: modules/sagemaker/sagemaker-model-cicd
+targetAccount: tooling
+parameters:
+  - name: infra-repo
+    value:
+      host: codecommit
+      name: cdk-infra-repo
+  - name: model-build-repo
+    value:
+      host: codecommit
+      name: model-build-repo
+  - name: deployment-groups
+    value:
+      # dev deployment group with experimentation on dev dataset
+      - name: experimenation-deployment-group
+        sourceBranch: dev
+        buildEnvironment:
+          name: experiment-env1
+          account: '123456789012'
+          region: us-east-1
+          type: dev
+        deployEnvironments:
+          - name: experiment-env2
+            account: '123456789013'
+            type: dev
+          - name: experiment-env3
+            account: '123456789014'
+            type: dev
+      # prod deployment group with retraining on prod dataset
+      - name: prod-deployment-group
+        sourceBranch: main
+        buildEnvironment:
+          name: prod-train-env
+          account: '123456789015'
+          region: us-east-1
+          type: preprod
+        deployEnvironments:
+          - name: prod-approval-env
+            account: '123456789016'
+            type: preprod
+          - name: prod-env
+            account: '123456789017'
+            type: prod
+```
+
+### Module Metadata Outputs
+
+#### Output Example

modules/sagemaker/sagemaker-model-cicd/bin/sagemaker-model-cicd.ts

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+import * as cdk from 'aws-cdk-lib';
+import { AwsSolutionsChecks } from 'cdk-nag';
+import 'source-map-support/register';
+import { MLOpsCodePipelineStack } from '../lib';
+import {
+  getModuleInfo,
+  getModuleParameters,
+  loadSeedFarmerEnvVars,
+} from './seedfarmer-parameters';
+
+// load seedfarmer environment variables from cache for codepipeline deployments
+loadSeedFarmerEnvVars();
+
+const account = process.env.CDK_DEFAULT_ACCOUNT;
+const region = process.env.CDK_DEFAULT_REGION;
+const { sfProjectName, sfDeploymentName, sfModuleName } = getModuleInfo();
+const moduleParameters = getModuleParameters();
+
+const app = new cdk.App();
+const stackName = moduleParameters.projectName;
+new MLOpsCodePipelineStack(app, stackName, {
+  env: {
+    account,
+    region,
+  },
+  // pass the tags in props so they can shared across other stacks deployed in codepipeline
+  tags: {
+    SeedFarmerProjectName: sfProjectName,
+    SeedFarmerDeploymentName: sfDeploymentName,
+    SeedFarmerModuleName: sfModuleName,
+  },
+  ...moduleParameters,
+});
+
+cdk.Aspects.of(app).add(new AwsSolutionsChecks());
+
+app.synth();

modules/sagemaker/sagemaker-model-cicd/bin/seedfarmer-parameters.ts

@@ -0,0 +1,74 @@
+import { fromError } from 'zod-validation-error';
+import { MLOpsCodePipelinePropsSchema } from '../lib';
+
+/**
+ * Load SeedFarmer environment variables from cache for CodePipeline deployments initiated externally,
+ * such as updates to model build or infrastructure repositories.
+ * These variables are stored in the CodePipeline build project following the initial build by SeedFarmer.
+ */
+export function loadSeedFarmerEnvVars() {
+  // check if this build is running from SeedFarmer codebuild project
+  if (
+    process.env.SEEDFARMER_PROJECT_NAME &&
+    process.env.SEEDFARMER_DEPLOYMENT_NAME &&
+    process.env.SEEDFARMER_MODULE_NAME
+  ) {
+    // nothing to do, environment variables are already set by SeedFarmer
+    return;
+  }
+
+  // if here, this build is running from codepipeline build step
+  console.log(
+    'Missing SeedFarmer environment variables. Checking cached environments from prior build',
+  );
+  const cachedEnv = process.env.CACHED_SEEDFARMER_ENV;
+  if (!cachedEnv || cachedEnv === '{}') {
+    // if here, running `cdk synth` directly instead of `seedfarmer apply`
+    throw new Error('Missing SeedFarmer module parameters');
+  }
+  const seedFarmerEnv = JSON.parse(cachedEnv);
+  process.env = { ...process.env, ...seedFarmerEnv };
+}
+
+export function getSeedFarmerParamter(name: string) {
+  const envVar = `SEEDFARMER_PARAMETER_${name}`;
+  const value = process.env[envVar];
+  if (!value) {
+    throw new Error(`Missing SeedFarmer parmeter: ${name}`);
+  }
+  try {
+    return JSON.parse(value);
+  } catch (e) {
+    return value;
+  }
+}
+
+export function getModuleInfo() {
+  const sfProjectName = process.env.SEEDFARMER_PROJECT_NAME!;
+  const sfDeploymentName = process.env.SEEDFARMER_DEPLOYMENT_NAME!;
+  const sfModuleName = process.env.SEEDFARMER_MODULE_NAME!;
+  return { sfProjectName, sfDeploymentName, sfModuleName };
+}
+
+export function getModuleParameters() {
+  const { sfProjectName, sfDeploymentName, sfModuleName } = getModuleInfo();
+  const mlopsProjectName = `${sfProjectName}-${sfDeploymentName}-${sfModuleName}`;
+  const infraRepo = getSeedFarmerParamter('INFRA_REPO');
+  const modelBuildRepo = getSeedFarmerParamter('MODEL_BUILD_REPO');
+  const deploymentGroups = getSeedFarmerParamter('DEPLOYMENT_GROUPS');
+
+  // validate parameters
+  try {
+    return MLOpsCodePipelinePropsSchema.parse({
+      projectName: mlopsProjectName,
+      infraRepo,
+      modelBuildRepo,
+      deploymentGroups,
+    });
+  } catch (err) {
+    const validationError = fromError(err, {
+      prefix: 'Invalid SeedFarmer module parameters',
+    });
+    throw validationError;
+  }
+}

modules/sagemaker/sagemaker-model-cicd/cdk.json

@@ -0,0 +1,67 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/sagemaker-model-cicd.ts",
+  "watch": {
+    "include": ["**"],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/core:target-partitions": ["aws", "aws-cn"],
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
+    "@aws-cdk/aws-iam:standardizedServicePrincipals": true,
+    "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
+    "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
+    "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
+    "@aws-cdk/aws-route53-patters:useCertificate": true,
+    "@aws-cdk/customresources:installLatestAwsSdkDefault": false,
+    "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
+    "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
+    "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
+    "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
+    "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
+    "@aws-cdk/aws-redshift:columnId": true,
+    "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
+    "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
+    "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
+    "@aws-cdk/aws-kms:aliasNameRef": true,
+    "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
+    "@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
+    "@aws-cdk/aws-efs:denyAnonymousAccess": true,
+    "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true,
+    "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true,
+    "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true,
+    "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true,
+    "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true,
+    "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true,
+    "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true,
+    "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true,
+    "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true,
+    "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true,
+    "@aws-cdk/aws-eks:nodegroupNameAttribute": true,
+    "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true,
+    "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true,
+    "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false
+  }
+}