Issue 10: Move state machine to terraform module (#11)

* step tf terraform dir * iam step in tf dir * move big module definition * changed resource vars * update CHANGELOG * reload, build * rename tf file and clear outputs.tf * yml file * readme: terraform and Module setup in updated cumulus deploy * readme: 0.1.1 * statemachine definition for bignbit internal terraform * add bignbit workflow to svc * Update browse_image_workflow.tf * remove var. duplicate * Update README.md * use templatefile for state machine definition * try the cumulus workflow option * comment 1 * switch back to isolated module test * prep invoke sm definition * big workflow, 2 * restore step.tf * init -upgrade * state_machine_definition * restore to deployable BrowseImageWorkflow, without cumulus * run. * iam role * readme * data source for definition * Implement the template as an output of the bignbit module (#13) * Try outputing rendered template * Update manual build handling * fix typo in environment * permission magic? * cleanup --------- Co-authored-by: Frank Greguska <89428916+frankinspace@users.noreply.github.com>
podaac · Mar 25, 2024 · b5d8e32 · b5d8e32
1 parent 51a2d4f
commit b5d8e32
Show file tree

Hide file tree

Showing 18 changed files with 1,656 additions and 987 deletions.
diff --git a/.github/workflows/cicd-pipeline.yml b/.github/workflows/cicd-pipeline.yml
@@ -43,50 +43,66 @@ jobs:
       pyproject_name: ${{ steps.poetry-build.outputs.pyproject_name }}
       tf_module_artifact_name: ${{ steps.poetry-build.outputs.tf_module_artifact_name }}
     steps:
-      - uses: getsentry/action-github-app-token@v2.0.0
+      - uses: getsentry/action-github-app-token@v3
         name: CICD Token
         id: cicd-app
         with:
           app_id: ${{ secrets.CICD_APP_ID }}
           private_key: ${{ secrets.CICD_APP_PRIVATE_KEY }}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
           token: ${{ steps.cicd-app.outputs.token }}
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Install Poetry
-        uses: abatilo/actions-poetry@v2
+        uses: abatilo/actions-poetry@v3
         with:
           poetry-version: ${{ env.POETRY_VERSION }}
-      - name: Get version
+      - name: Setup a local virtual environment
+        run: |
+          poetry config virtualenvs.create true --local
+          poetry config virtualenvs.in-project true --local
+      - uses: actions/cache@v4
+        name: Define a cache for the virtual environment based on the dependencies lock file
+        with:
+          path: ./.venv
+          key: venv-${{ hashFiles('poetry.lock') }}
+      - name: Get pre-build version
         id: get-version
         run: |
           echo "current_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT
           echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV
+      - name: Manual Build
+        # If triggered by workflow dispatch, no version bump
+        if: github.event_name == 'workflow_dispatch'
+        id: manual
+        run: |
+          echo "TARGET_ENV_UPPERCASE=${{ github.event.inputs.venue }}" >> $GITHUB_ENV
       - name: Bump pre-alpha version
         # If triggered by push to a non-tracked branch
         if: |
-            github.ref != 'refs/heads/develop' &&
-            github.ref != 'refs/heads/main'    &&
-            !startsWith(github.ref, 'refs/heads/release')
+          github.ref != 'refs/heads/develop' &&
+          github.ref != 'refs/heads/main'
         run: |
           new_ver="${{ steps.get-version.outputs.current_version }}+$(git rev-parse --short ${GITHUB_SHA})"
           poetry version $new_ver
-          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
           echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV
       - name: Bump alpha version
         # If triggered by push to the develop branch
-        if: ${{ github.ref == 'refs/heads/develop' }}
+        if: |
+          github.ref == 'refs/heads/develop' &&
+          steps.manual.conclusion == 'skipped'
         id: alpha
         run: |
           poetry version prerelease
-          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
           echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV
       - name: Bump rc version
         # If triggered by push to a release branch
-        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
+        if: |
+          startsWith(github.ref, 'refs/heads/release/') &&
+          steps.manual.conclusion == 'skipped'
         id: rc
         env:
           # True if the version already has a 'rc' pre-release identifier
@@ -97,18 +113,22 @@ jobs:
           else
             poetry version ${GITHUB_REF#refs/heads/release/}rc1
           fi
-          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
           echo "TARGET_ENV_UPPERCASE=UAT" >> $GITHUB_ENV
       - name: Release version
         # If triggered by push to the main branch
-        if: ${{ startsWith(github.ref, 'refs/heads/main') }}
+        if: |
+          startsWith(github.ref, 'refs/heads/main') &&
+          steps.manual.conclusion == 'skipped'
         id: release
         run: |
           poetry version major
-          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
           echo "TARGET_ENV_UPPERCASE=OPS" >> $GITHUB_ENV
+      - name: Get install version
+        # Get the version of the software being installed and save it as an ENV var
+        run: |
+          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
       - name: Install conda
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           activate-environment: bignbit
           environment-file: conda-environment.yaml
@@ -122,14 +142,14 @@ jobs:
       - name: Test and coverage
         run: |
           poetry run pytest --junitxml=build/reports/pytest.xml --cov=bignbit --cov-report=xml:build/reports/coverage.xml tests/
-      - uses: hashicorp/setup-terraform@v2
+      - uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: ${{ env.TERRAFORM_VERSION }}
           terraform_wrapper: false
       - name: Validate Terraform
         working-directory: terraform
         run: |
-          terraform init -backend=false
+          terraform init -backend=false -upgrade
           terraform validate -no-color
       - name: SonarCloud Scan
         uses: sonarsource/sonarcloud-github-action@master
@@ -154,13 +174,15 @@ jobs:
           echo "the_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT
           echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_OUTPUT
           echo "tf_module_artifact_name=$(poetry version | awk '{print $1}')-${{ env.software_version }}-cumulus-tf" >> $GITHUB_OUTPUT
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.poetry-build.outputs.pyproject_name }}-dist
           path: dist/*
+      - name: Rename TF Module directory
+        run: mv terraform ${{ steps.poetry-build.outputs.tf_module_artifact_name }}
       - name: Zip artifact for deployment
-        run: zip ${{ steps.poetry-build.outputs.tf_module_artifact_name }}.zip terraform/* -r
-      - uses: actions/upload-artifact@v3
+        run: zip ${{ steps.poetry-build.outputs.tf_module_artifact_name }}.zip ${{ steps.poetry-build.outputs.tf_module_artifact_name }}/* -r
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.poetry-build.outputs.tf_module_artifact_name }}
           path: ${{ steps.poetry-build.outputs.tf_module_artifact_name }}.zip
@@ -209,24 +231,24 @@ jobs:
       THE_VERSION: ${{ needs.build.outputs.version }}
       PYPROJECT_NAME: ${{ needs.build.outputs.pyproject_name }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
       - name: Log in to the Container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ github.repository }}/${{env.PYPROJECT_NAME}}
           tags: |
             type=pep440,pattern={{version}},value=${{ env.THE_VERSION }}
       - name: Build and push Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: docker/Dockerfile
@@ -242,7 +264,7 @@ jobs:
     name: Deploy
     needs: [build, docker]
     runs-on: ubuntu-latest
-    environment: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.venue || needs.build.outputs.deploy_env }}
+    environment: ${{ needs.build.outputs.deploy_env }}
     env:
       THE_VERSION: ${{ needs.build.outputs.version }}
       CONTAINER_IMAGE_URI: ${{ needs.docker.outputs.container_image_uri }}
@@ -253,10 +275,10 @@ jobs:
       startsWith(github.ref, 'refs/heads/release') ||
       github.event_name == 'workflow_dispatch'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
-      - uses: hashicorp/setup-terraform@v2
+      - uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: ${{ env.TERRAFORM_VERSION }}
           terraform_wrapper: false
@@ -273,7 +295,7 @@ jobs:
       - name: Validate Terraform
         working-directory: examples/cumulus-tf
         run: |
-          terraform init -backend=false
+          terraform init -backend=false -upgrade
           terraform validate -no-color
 
       - name: Deploy to venue
@@ -282,4 +304,4 @@ jobs:
         env:
           AWS_DEFAULT_REGION: us-west-2
         run: |
-          ./bin/deploy.sh --app-version ${{ env.THE_VERSION }} --tf-venue ${{ vars.TF_VENUE }} --lambda_container_image_uri ${{ env.CONTAINER_IMAGE_URI }}
+          ./bin/deploy.sh --app-version ${{ env.THE_VERSION }} --tf-venue ${{ vars.TF_VENUE }} --lambda_container_image_uri ${{ env.CONTAINER_IMAGE_URI }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `bignbit` Issue #3: [Update terraform mock deployment of cumulus module to services accounts](https://github.com/podaac/bignbit/issues/3),
 
 ### Changed
+- `bignbit` Issue #10: [Move combined big and pobit state machine into terraform module](https://github.com/podaac/bignbit/issues/10),
 - `bignbit` Issue #6: [BIG terraform failing in SWOT venues due to long lambda name](https://github.com/podaac/bignbit/issues/6),
 
 

diff --git a/README.md b/README.md
@@ -1,7 +1,8 @@
 # About
 
-bignbit combines the Browse Image Generator (BIG) and the PO.DAAC Browse Image Transfer (pobit) modules
+bignbit is a Cumulus module that can be installed as a post-ingest workflow to generate browse imagery via Harmony and then transfer that imagery to GIBS.
 
+See an example of how to use this cumulus module in [browse_image_workflow.tf](/examples/cumulus-tf/browse_image_workflow.tf).
 
 # Assumptions
  - Using `ContentBasedDeduplication` strategy for GITC input queue
@@ -13,4 +14,4 @@ bignbit combines the Browse Image Generator (BIG) and the PO.DAAC Browse Image T
 2. Run `conda env create -f conda-environment.yml` to install GDAL
 3. Activate the bignbit conda environment `conda activate bignbit`
 4. Install python package and dependencies `poetry install`
-5. Verify tests pass `poetry run pytest tests/`
+5. Verify tests pass `poetry run pytest tests/`
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -17,7 +17,7 @@ RUN apt-get update && \
   unzip \
   libcurl4-openssl-dev
 
-RUN adduser --quiet --disabled-password --shell /bin/sh --home /home/dockeruser --gecos "" --uid 1000 dockeruser
+RUN adduser --quiet --disabled-password --shell /bin/sh --home /home/dockeruser --gecos "" --uid 993 dockeruser
 USER dockeruser
 RUN curl -sSL https://install.python-poetry.org | python3 -
 
@@ -36,15 +36,15 @@ RUN mkdir -p "${FUNCTION_DIR}" && \
 
 FROM osgeo/gdal:ubuntu-small-${GDAL_VERSION}
 ARG FUNCTION_DIR
-RUN adduser --quiet --disabled-password --shell /bin/sh --home /home/dockeruser --gecos "" --uid 1000 dockeruser
+RUN adduser --quiet --disabled-password --shell /bin/sh --home /home/dockeruser --gecos "" --uid 993 dockeruser
 USER dockeruser
 ENV HOME /home/dockeruser
 WORKDIR ${FUNCTION_DIR}
 
-COPY --chown=1000:1000 --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR}
+COPY --chown=dockeruser --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR}
 # (Optional) Add Lambda Runtime Interface Emulator and use a script in the ENTRYPOINT for simpler local runs
-ADD --chown=1000:1000 https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie /home/dockeruser/.local/bin/aws-lambda-rie
-COPY --chown=1000:1000 docker/docker-entrypoint.sh /home/dockeruser/.local/bin/
+ADD --chown=dockeruser https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie /home/dockeruser/.local/bin/aws-lambda-rie
+COPY --chown=dockeruser docker/docker-entrypoint.sh /home/dockeruser/.local/bin/
 RUN chmod 755 /home/dockeruser/.local/bin/aws-lambda-rie /home/dockeruser/.local/bin/docker-entrypoint.sh && \
     touch bignbit/__init__.py
 ENTRYPOINT [ "/home/dockeruser/.local/bin/docker-entrypoint.sh" ]

diff --git a/examples/README.md b/examples/README.md
@@ -0,0 +1,13 @@
+# BIGnBIT Module In Cumulus Deployment (cumulus-tf)
+- BIGnBIT Module used in `cumulus-deploy-tf/cumulus-tf`
+  - Module Name: `big_and_bit_module`
+    - Source ZIP: `https://github.com/podaac/bignbit/releases/download/0.1.1/bignbit-0.1.1-cumulus-tf.zip`
+    - lambda_container_image_uri: `ghcr.io/podaac/bignbit/bignbit:0.1.1`
+
+## Terraform ZIP
+
+- variables.tf
+- outputs.tf
+- main.tf
+- lambda_functions.tf
+- state_machine_definition.json
diff --git a/examples/cumulus-tf/.terraform.lock.hcl b/examples/cumulus-tf/.terraform.lock.hcl
diff --git a/examples/cumulus-tf/big.tf → examples/cumulus-tf/bignbit.tf b/examples/cumulus-tf/big.tf → examples/cumulus-tf/bignbit.tf
@@ -7,7 +7,6 @@ module "bignbit_module" {
     security_group_ids = []
     subnet_ids = []
     app_name = var.app_name
-    #lambda_container_image_uri = "ghcr.io/podaac/podaac-big:${var.app_version}"
     lambda_container_image_uri = var.lambda_container_image_uri
 
     default_tags = local.default_tags

diff --git a/examples/cumulus-tf/browse_image_workflow.tf b/examples/cumulus-tf/browse_image_workflow.tf
@@ -0,0 +1,21 @@
+# In a typical cumulus installation, this is how you would define the workflow:
+/*
+module "browse_image_workflow" {
+  source          = "https://github.com/nasa/cumulus/releases/download/v16.1.2/terraform-aws-cumulus-workflow.zip"
+  prefix          = var.prefix
+  name            = "BrowseImageWorkflow"
+  workflow_config = module.cumulus.workflow_config
+  system_bucket   = var.system_bucket
+  tags            = merge(local.tags, { application = "BrowseImageWorkflow" })
+
+  state_machine_definition = module.bignbit_module.workflow_definition
+}
+*/
+
+# This example is deployable without cumulus installed for the purpose of testing the module in isolation; so it does not use terraform-aws-cumulus-workflow.zip
+resource "aws_sfn_state_machine" "sfn_state_machine" {
+  name     = "${local.ec2_resources_name}-BrowseImageWorkflow"
+  role_arn = aws_iam_role.step.arn
+
+  definition = module.bignbit_module.workflow_definition
+}
diff --git a/examples/cumulus-tf/iam_step.tf b/examples/cumulus-tf/iam_step.tf
@@ -1,6 +1,5 @@
 data "aws_region" "current" {}
 
-
 data "aws_iam_policy_document" "states_assume_role_policy" {
   statement {
     actions = ["sts:AssumeRole"]
@@ -12,7 +11,7 @@ data "aws_iam_policy_document" "states_assume_role_policy" {
 }
 
 resource "aws_iam_role" "step" {
-  name                 = "${local.ec2_resources_name}-steprole"
+  name                 = "${local.ec2_resources_name}-sfn-steprole"
   assume_role_policy   = data.aws_iam_policy_document.states_assume_role_policy.json
   permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/NGAPShRoleBoundary"
 }