Merge branch 'main' into feature/deploy-pipeline-infra

.github/ISSUE_TEMPLATE/missing_integ_tests_issue.md

@@ -0,0 +1,6 @@
+---
+name: Missing Integration Tests
+title: "[CHORE] Module {{env.MODULE_PATH }} missing integration tests"
+labels: workflows, backlog
+assignees: ''
+---

.github/workflows/check-integration-tests.yml

@@ -0,0 +1,60 @@
+name: Check Modules for Integration Tests
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 */3 * * *"
+
+jobs:
+  get-modules:
+    name: Get Modules
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.get-modules.outputs.matrix }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Get modules
+        id: get-modules
+        run: |
+          MODULES=()
+          MODULES_WITHOUT_INTEG=()
+          for d in modules/*/* ; do
+              MODULES+=("$d")
+          done
+          for m in "${MODULES[@]}" ; do
+              if [ ! -d "${m}"/integ/ ]; then
+                echo "No Integration tests for ${m}"
+                MODULES_WITHOUT_INTEG+=("$m")
+              fi
+          done
+          echo "${MODULES_WITHOUT_INTEG[@]}"
+          # Create our json structure [{"module_path": "..."}]
+          MODULES_JSON=$(echo "${MODULES_WITHOUT_INTEG[@]}" | jq -R -s 'split(" ")' | jq '[ .[] | select(length > 0) ]' | jq 'map({"module_path": .})')
+          # Export the modules as json to the outputs
+          echo 'matrix<<EOF' >> $GITHUB_OUTPUT
+          echo $MODULES_JSON >> $GITHUB_OUTPUT
+          echo 'EOF' >> $GITHUB_OUTPUT
+
+  create-issues:
+    name: Create issue for ${{ matrix.modules.module_path }}
+    needs: get-modules
+    permissions:
+      contents: read
+      issues: write
+    strategy:
+      fail-fast: false
+      matrix:
+        modules: ${{ fromJson(needs.get-modules.outputs.matrix) }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: JasonEtco/create-an-issue@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MODULE_PATH: ${{ matrix.modules.module_path }}
+        with:
+          filename: .github/ISSUE_TEMPLATE/missing_integ_tests_issue.md
+          update_existing: false

.github/workflows/mwaa-requirements.check.yml

@@ -0,0 +1,41 @@
+name: MWAA Requirements Check
+
+on:
+  push:
+    branches: 
+      - "main"
+    paths:
+      - "data/mwaa/requirements/**"
+
+  pull_request:
+    branches:
+      - "main"
+      - "release/*"
+      - "stable"
+    paths:
+      - "data/mwaa/requirements/**"
+
+jobs:
+  install:
+    name: Install Requirements
+    runs-on: ubuntu-latest
+    env:
+      AIRFLOW_VERSION: 2.8.1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Update pip
+        run: pip install --upgrade pip
+      - name: Install Airflow and Dependencies
+        working-directory: ./data/mwaa/requirements/
+        run: |
+          tail -n +2 requirements.txt > requirements-no-constraint.txt
+          pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==$AIRFLOW_VERSION" \
+            --constraint constraint-requirements.txt \
+            --requirement requirements-no-constraint.txt

.github/workflows/one-click-template-lint.yml

@@ -19,6 +19,10 @@ on:
       - "requirements-dev.txt"
       - ".github/workflows/one-click-template-lint.yml"
 
+permissions:
+  contents: read
+  security-events: write
+
 jobs:
   lint:
     name: CFN Lint Template
@@ -38,5 +42,24 @@ jobs:
           set -x
           python -m pip install --upgrade pip
           pip install -r requirements-dev.txt
-      - name: Lint
+      - name: cfn-lint
         run: cfn-lint --template ./one-click-launch.yaml
+
+  nag:
+    name: CFN Nag Template
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: cfn-nag
+        uses: stelligent/cfn_nag@master
+        with:
+          input_path: ./one-click-launch.yaml
+          extra_args: -o sarif
+          output_path: cfn_nag.sarif
+      - uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: cfn_nag.sarif

CHANGELOG.md

@@ -12,8 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### **Added**
 
 - added `ray-on-eks`, and `manifests/ray-on-eks` manifests
-- Added a `sagemaker-model-monitoring-module` module with an example of data quality and model quality monitoring of a SageMaker Endpoint.
-- Added an option to enable data capture in the `sagemaker-endpoint-module`.
+- added a `sagemaker-model-monitoring-module` module with an example of data quality, model quality, model bias, and model explainability monitoring of a SageMaker Endpoint
+- added an option to enable data capture in the `sagemaker-endpoint-module`
+- added a `personas` example module to deploy various roles required for an AI/ML project 
 
 ### **Changed**
 - remove explicit module manifest account/region mappings from `fmops-qna-rag`
@@ -22,6 +23,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - changed encryption for each bucket to KMS_MANAGED
 - refactor `airflow-dags` module to use Pydantic
 - fix inputs for `bedrock-finetuning` module not working
+- add `retention-type` argument for the bucket in the `bedrock-finetuning` module
+- fix broken dependencies for `examples/airflow-dags`
+- use `add_dependency` to avoid deprecation warnings from CDK
+- various typo fixes
+- various clean-ups to the SageMaker Service Catalog templates
+- fix opensearch removal policy
 
 ## v1.2.0
 

DEPLOYMENT.md

@@ -29,7 +29,7 @@ export PRIMARY_ACCOUNT=XXXXXXXXXXXX
 export ADMIN_ROLE_ARN=arn:aws:iam::XXXXXXXXXXXX:role/XXXXX
 ```
 
-5. Bbootstrap the CDK environment (one time per region) with CDK V2. Asuming you are deploying in `us-east-1`:
+5. Bootstrap the CDK environment (one time per region) with CDK V2. Assuming you are deploying in `us-east-1`:
 ```
 cdk bootstrap aws://${PRIMARY_ACCOUNT}/us-east-1
 ```
@@ -44,7 +44,7 @@ seedfarmer bootstrap toolchain --project aiops --trusted-principal ${ADMIN_ROLE_
 
 ## Deployment
 
-Pick the manifest to deploy. Manifests are located in `mainfests/` directory. For example, to deploy SageMaker mlops manifests, run:
+Pick the manifest to deploy. Manifests are located in `manifests/` directory. For example, to deploy SageMaker mlops manifests, run:
 
 !Note: if you are deploying into a region different from `us-east-1`, change the `regionMappings` in `deployment.yaml`.
 ```