.github/workflows/build.yml

# Build Pipeline for hitide-backfill-lambdas
name: Build
# Controls when the workflow will run
on:
  # Triggers the workflow on push events
  push:
    branches: [ develop, release/**, main, feature/**, issue/**, issues/**, dependabot/** ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  # First job in the workflow installs and verifies the software
  build:
    name: Build, Test, Verify, Publish
    # The type of runner that the job will run on
    runs-on: ubuntu-latest
    steps:
      #########################################################################
      # Environment Setup
      #########################################################################
      # NOTE: This step is platform-specific
      # Checks out this repository and sets up the build/test environment with 
      # gradle
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install Poetry
        uses: abatilo/actions-poetry@v3
        with:
          poetry-version: 1.8.1

      #########################################################################
      # Versioning (featuring weird gradle output work-arounds)
      #########################################################################
      # NOTE: This step is platform-specific
      # Retrieve version information for use in the other versioning steps
      - name: Get version
        id: get-version
        run: |
          echo "the_service=${{ github.event.repository.name }}" >> $GITHUB_ENV
          echo "the_env=$(printenv)" >> $GITHUB_ENV
          echo "${{ github.event.repository.name }}"
          echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV
          poetry version > .temp_version.out
          cat .temp_version.out
          the_version=$(cat .temp_version.out |grep -v Downloading |grep -v '%' |sed -e 's/podaac-hitide-backfill-tool *//')
          rm .temp_version.out
          echo "old_version=$the_version" >> $GITHUB_ENV
          echo "the_version=$the_version" >> $GITHUB_ENV
          echo "Initial Version: $the_version"

      # Pre-Alpha Logic - Use the project version number and add the short hash
      # to it
      - name: Bump pre-alpha version
        # If triggered by push to a feature branch
        if: |
          startsWith(github.ref, 'refs/heads/issue') || 
          startsWith(github.ref, 'refs/heads/dependabot/') || 
          startsWith(github.ref, 'refs/heads/feature/')
        # At pre-alpha, append git-commit to version, set it into gradle
        # property, read the version out and set to build_service_version
        run: |
          the_version=$(echo "${{ env.the_version }}" | sed -e "s/-alpha.*//g")
          the_version=$(echo "$the_version" | sed -e "s/-rc.*//g")
          new_version="${the_version}+$(git rev-parse --short HEAD)"
          echo "the_version=${new_version}" >> $GITHUB_ENV
          echo "software_version=${new_version}" >> $GITHUB_ENV
          echo "new_version=${new_version}" >> $GITHUB_ENV
          echo "Github REF: ${{ github.ref }}"
          echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV

      # Alpha Logic - Use the project version number and add -alpha.1 or bump
      # alpha number
      - name: Bump alpha version
        env:
          VERSION: ${{ env.the_version }}
        # If triggered by push to the develop branch
        if: ${{ github.ref == 'refs/heads/develop' }}
        run: |
          if [[ ${VERSION} == *"-alpha"* ]]; then
            alpha_number=$(echo "${VERSION}" | sed -e "s/^.*-alpha.//g")
            alpha_number=$(echo "$alpha_number" | sed -e "s/-rc.*//g")
            alpha_number=$((alpha_number+1))
            the_version=$(echo "$the_version" | sed -e "s/-alpha.*//g")
            the_version=$(echo "$the_version" | sed -e "s/-rc.*//g")
            the_version="${the_version}-alpha.$alpha_number"
            echo "software_version=${the_version}" >> $GITHUB_ENV
            echo "the_version=${the_version}" >> $GITHUB_ENV
          else
            the_version="${{ env.the_version }}-alpha.1"
            echo "software_version=${the_version}" >> $GITHUB_ENV
            echo "the_version=${the_version}" >> $GITHUB_ENV
          fi
          echo "new_version=${the_version}" >> $GITHUB_ENV
          echo "venue=sit" >> $GITHUB_ENV
          echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV

      # Release Candidate Logic - Remove -alpha* and add -rc.1, or bump the rc
      # number
      - name: Bump rc version
        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
        env:
          VERSION: ${{ env.the_version }}
          COMMIT_VERSION: ${{ github.ref }}
        run: |
          commit_version=$COMMIT_VERSION
          commit_version=$(echo "${commit_version}" |sed -e "s/^.*\///g")
          commit_version=$(echo "${commit_version}" |sed -e "s/-alpha.*//g")
          commit_version=$(echo "${commit_version}" |sed -e "s/-rc.*//g")
          echo "COMMIT VERSION: $commit_version"
          file_version=${VERSION}
          file_version=$(echo "${file_version}" |sed -e "s/-alpha.*//g")
          file_version=$(echo "${file_version}" |sed -e "s/-rc.*//g")
          echo "FILE VERSION: $file_version"
          if [[ "$commit_version" != "$file_version" ]]; then
            echo "Commit version and file version are different, using commit version"
            VERSION=$commit_version
          fi
          if [[ ${VERSION} == *"-rc"* ]]; then
            echo "Bumping up the release candidate number from ${VERSION}"
            rc_number=$(echo "${VERSION}" | sed -e "s/^.*-rc.//g")
            rc_number=$(echo "${rc_number}" | sed -e "s/-alpha.*//g")
            rc_number=$((rc_number+1))
            the_version=$(echo "$the_version" | sed -e "s/-rc.*//g")
            the_version=$(echo "$the_version" | sed -e "s/-alpha.*//g")
            VERSION="${the_version}-rc.${rc_number}"
          else
            echo "Initializing the first release candidate for ${VERSION}"
            VERSION=$(echo "${VERSION}" |sed -e "s/-alpha.*//g")
            VERSION="${VERSION}-rc.1"
          fi
          echo "software_version=${VERSION}" >> $GITHUB_ENV
          echo "the_version=${VERSION}" >> $GITHUB_ENV
          echo "new_version=${VERSION}" >> $GITHUB_ENV
          echo "venue=uat" >> $GITHUB_ENV
          echo "TARGET_ENV_UPPERCASE=UAT" >> $GITHUB_ENV


      # Release Logic
      - name: Release version
        # If triggered by push to the main branch
        if: ${{ startsWith(github.ref, 'refs/heads/main') }}
        env:
          VERSION: ${{ env.the_version }}
        # Remove -rc.* from end of version string
        run: |
          software_version=$(echo "${VERSION}" | sed -e s/-rc.*//g)
          software_version=$(echo "${software_version}" | sed -e s/-alpha.*//g)
          echo "software_version=$software_version" >> $GITHUB_ENV
          echo "new_version=$software_version" >> $GITHUB_ENV
          echo "the_version=$software_version" >> $GITHUB_ENV
          echo "venue=ops" >> $GITHUB_ENV
          echo "TARGET_ENV_UPPERCASE=OPS" >> $GITHUB_ENV


      #########################################################################
      # Versioning Summary
      #########################################################################
      - name: Versioning Summary
        run: |
          echo "the_service: ${{ env.the_service }}"
          echo "old version : ${{ env.old_version }}"
          echo "new version : ${{ env.new_version }}"
          echo "the_env: ${{ env.the_env }}"
          echo "software_version: ${{ env.software_version }}"
          echo "GITHUB REF: ${{ github.ref }}"
          echo "VENUE: ${{ env.venue }}"
          echo "Target Env Uppercase: ${{ env.TARGET_ENV_UPPERCASE }}"

          original_env_value="${TARGET_ENV_UPPERCASE}"
          lowercase_value=$(echo "${original_env_value}" | tr '[:upper:]' '[:lower:]')
          echo "TARGET_ENV_LOWERCASE=${lowercase_value}" >> $GITHUB_ENV

      # NOTE: This step is platform-specific
      # Update the version number in the application package itself
      - name: Update version number in the application package
        run: |
          poetry version ${{ env.the_version }}


      #########################################################################
      # Install & Test & Snyk
      #########################################################################
      # NOTE: This step is platform-specific
      # These are gradle-specific steps for installing the application
      - name: Install Software
        run: |
          pip install pylint
          pip install pytest
          poetry build
          poetry install

      # This is where tests go
      - name: Run Poetry Tests
        run: |
          poetry run pylint podaac
          poetry run flake8 podaac
          poetry run pytest --junitxml=build/reports/pytest.xml --cov=podaac/ --cov-report=html -m "not aws and not integration" tests/

      - name: Run Snyk as a blocking step
        uses: snyk/actions/python@master
        env:
            SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          command: test
          args: >
              --org=${{ secrets.SNYK_ORG_ID }}
              --project-name=${{ github.repository }}
              --severity-threshold=high
              --fail-on=all

      - name: Run Snyk on Python
        uses: snyk/actions/python@master
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          command: monitor
          args: >
            --org=${{ secrets.SNYK_ORG_ID }}
            --project-name=${{ github.repository }}


      #########################################################################
      # Publish new version numbers
      #########################################################################
      
      - name: Quick check for changes
        id: check_changes
        if: |
          github.ref == 'refs/heads/develop' ||
          github.ref == 'refs/heads/main'    ||
          startsWith(github.ref, 'refs/heads/release')
        run: |
          if [ -n "$(git status --porcelain)" ]; then
            echo "changes=true" >> $GITHUB_OUTPUT
          else
            echo "changes=false" >> $GITHUB_OUTPUT
          fi
      
      - name: Commit Version Bump
        # If building develop, a release branch, or main then we commit the version bump back to the repo
        if: steps.check_changes.outputs.changes == 'true'
        run: |
          git config user.name "${GITHUB_ACTOR}"
          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
          git commit -am "/version ${{ env.the_version }}"
          git push

      - name: Push Tag
        env:
          VERSION: ${{ env.the_version }}
        if: |
          github.ref == 'refs/heads/develop' ||
          github.ref == 'refs/heads/main'    ||
          startsWith(github.ref, 'refs/heads/release')
        run: |
          git config user.name "${GITHUB_ACTOR}"
          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
          git tag -a "${VERSION}" -m "Version ${VERSION}"
          git push origin "${VERSION}"


      #########################################################################
      # Publish to pypi.org
      #########################################################################
      - name: Publish to test.pypi.org
        id: pypi-test-publish
        if: |
          github.ref == 'refs/heads/develop' ||
          startsWith(github.ref, 'refs/heads/release')
        env:
          POETRY_PYPI_TOKEN_TESTPYPI: ${{secrets.TEST_PYPI_API_TOKEN}}
        run: |
          poetry config repositories.testpypi https://test.pypi.org/legacy/
          poetry publish -r testpypi

      - name: Publish to pypi.org
        if: ${{ github.ref == 'refs/heads/main' }}
        id: pypi-publish
        env:
          POETRY_PYPI_TOKEN_PYPI: ${{secrets.PYPI_API_TOKEN}}
        run: |
          poetry publish --skip-existing

      ## Due to observed delays between upload and availability, wait for the package to become available
      - name: Wait for package
        if: |
          steps.pypi-test-publish.conclusion == 'success' ||
          steps.pypi-publish.conclusion == 'success'
        run: |
          pip install tenacity logging 
          python3 ${GITHUB_WORKSPACE}/.github/workflows/wait-for-pypi.py ${{env.pyproject_name}}==${{ env.software_version }}


      #########################################################################
      # Deploy to AWS via Terraform
      #########################################################################
      - name: Deploy Env Override
        if: |
          github.event.head_commit.message == '/deploy sit' ||
          github.event.head_commit.message == '/deploy uat'
        run: |
          message="${{ github.event.head_commit.message }}"
          trimmed_message=${message:1}  # Remove leading slash
          override_env=$(echo "$trimmed_message" | grep -oE '[^[:space:]]+$')
          override_env_upper=$(echo "$trimmed_message" | awk '{print toupper($NF)}')
          echo "THE_ENV=${override_env}" >> $GITHUB_ENV
          echo "TARGET_ENV_LOWERCASE=${override_env}" >> $GITHUB_ENV
          echo "TARGET_ENV_UPPERCASE=${override_env_upper}" >> $GITHUB_ENV

      - uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: 1.5.3

      - name: Deploy Terraform
        if: |
          github.ref == 'refs/heads/develop' ||
          github.ref == 'refs/heads/main'    ||
          startsWith(github.ref, 'refs/heads/release') ||
          github.event.head_commit.message == '/deploy sit' ||
          github.event.head_commit.message == '/deploy uat'
        working-directory: terraform-deploy/
        env:
          AWS_ACCESS_KEY_ID:       ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
          AWS_SECRET_ACCESS_KEY:   ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
          AWS_ACCOUNT_ID:          ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
          AWS_DEFAULT_REGION:      us-west-2

          TF_VAR_permissions_boundary_arn: ${{ secrets[format('PERMISSIONS_BOUNDARY_ARN_{0}', env.TARGET_ENV_UPPERCASE)] }}
          TF_VAR_buckets_name: ${{ secrets[format('BUCKET_{0}', env.TARGET_ENV_UPPERCASE)] }}
          TF_VAR_system_bucket: ${{ secrets[format('SYSTEM_BUCKET_{0}', env.TARGET_ENV_UPPERCASE)] }}
          TF_VAR_dmrpp_url: ${{ secrets.DMRPP_URL }}
          TF_VAR_aws_security_group_ids: ${{ secrets[format('SECURITY_GROUP_IDS_{0}', env.TARGET_ENV_UPPERCASE)] }}

        run: |
          curl -L -o metadata-aggregator.zip https://github.com/podaac/cumulus-metadata-aggregator/releases/download/v8.7.0-alpha.6-SNAPSHOT/cumulus-metadata-aggregator-8.7.0-alpha.6-SNAPSHOT.zip
          python3 override.py "${{ env.TARGET_ENV_LOWERCASE }}"
          source bin/config.sh ${{ env.TARGET_ENV_LOWERCASE }}
          terraform plan -var-file=tfvars/"${{ env.TARGET_ENV_LOWERCASE }}".tfvars -var="app_version=${{ env.the_version }}" -out="tfplan"
          terraform apply -auto-approve tfplan