upload-and-deploy-to-prod-main.yaml

name: upload-and-deploy-to-prod-main

on:
  push:
    tags:
      - "v[0-9]+.[0-9]+.[0-9]+"

env:
  NAMESPACE: main
  PYTHON_VERSION: 3.9

jobs:

  pre-commit:
    name: Run pre-commit hooks against all files
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
      - uses: pre-commit/action@v3.0.0


  upload-files:
    name: Upload files to S3 bucket in prod/main
    runs-on: ubuntu-latest
    needs: pre-commit
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read
    environment: prod
    steps:

      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: GitHubActions-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Copy files to templates bucket
        run: python src/scripts/manage_artifacts/artifacts.py --upload --namespace $NAMESPACE --cfn_bucket ${{ vars.CFN_BUCKET }}


  sceptre-deploy-main:
    name: Deploys to prod/main using sceptre
    runs-on: ubuntu-latest
    needs: [pre-commit, upload-files]
    environment: prod
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read

    steps:
      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: GitHubActions-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Create directory for remote sceptre templates
        run: mkdir -p templates/remote/

      - name: Deploy sceptre stacks to prod/main
        run: pipenv run sceptre --var "namespace=${{ env.NAMESPACE }}" launch prod --yes

      - name: Delete preexisting S3 event notification for this namespace
        uses: gagoar/invoke-aws-lambda@v3
        with:
          AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
          AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
          REGION: ${{ env.AWS_REGION }}
          FunctionName: ${{ env.NAMESPACE }}-S3EventConfig
          Payload: '{"RequestType": "Delete"}'
          LogType: Tail

      - name: Create S3 event notification for this namespace
        uses: gagoar/invoke-aws-lambda@v3
        with:
          AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
          AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
          REGION: ${{ env.AWS_REGION }}
          FunctionName: ${{ env.NAMESPACE }}-S3EventConfig
          Payload: '{"RequestType": "Create"}'
          LogType: Tail

  deploy-snowflake-main:
    name: Deploy Snowflake resources
    needs: sceptre-deploy-main
    runs-on: ubuntu-latest
    env:
      PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PASSPHRASE }}
    steps:
      - uses: actions/checkout@v3

      - name: Configure Snowflake connection
        run: |
          # Create temporary files for config.toml and our private key
          config_file=$(mktemp)
          private_key_file=$(mktemp)

          # Write to the private key file
          echo "${{ secrets.SNOWFLAKE_PRIVATE_KEY }}" > $private_key_file

          # Write to config.toml file
          echo 'default_connection_name = "recover"' >> $config_file
          echo '[connections.recover]' >> $config_file
          echo "account = \"${{ vars.SNOWFLAKE_ACCOUNT }}\"" >> $config_file
          echo "user = \"${{ vars.SNOWFLAKE_USER }}\"" >> $config_file
          echo "role = \"${{ vars.SNOWFLAKE_ROLE }}\"" >> $config_file
          echo 'warehouse = "RECOVER_XSMALL"' >> $config_file
          echo 'authenticator = "SNOWFLAKE_JWT"' >> $config_file
          echo "private_key_path = \"$private_key_file\"" >> $config_file

          # Write config.toml path to global environment
          echo "SNOWFLAKE_CONFIG_PATH=$config_file" >> $GITHUB_ENV

      - name: Configuration file information
        run: |
          echo "Snowflake configuration is located at $SNOWFLAKE_CONFIG_PATH"
          cat $SNOWFLAKE_CONFIG_PATH

      - name: Install Snowflake CLI
        uses: Snowflake-Labs/snowflake-cli-action@v1
        with:
          default-config-file-path: ${{ env.SNOWFLAKE_CONFIG_PATH }}

      - name: Test Snowflake connection
        run: |
          snow --version
          snow connection test

      - name: Deploy Snowflake objects
        run: |
          snow sql \
            -D "git_branch=main" \
            -D "environment=main" \
            -f snowflake/objects/deploy.sql

  sts-access-test:
    name: Runs STS access tests on prod synapse folders
    runs-on: ubuntu-latest
    needs: sceptre-deploy-main
    environment: prod
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read

    steps:
      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: ${{ github.event.repository.name }}-${{ github.run_id }}-nonglue-unit-tests
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Test prod synapse folders for STS access with pytest
        run: >
          pipenv run python -m pytest tests/test_setup_external_storage.py
          --test-bucket recover-input-data
          --test-synapse-folder-id syn51714264
          --namespace $NAMESPACE
          --test-sts-permission read_only
          -v

          pipenv run python -m pytest tests/test_setup_external_storage.py
          --test-bucket recover-processed-data
          --test-synapse-folder-id syn51406699
          --namespace $NAMESPACE/parquet
          --test-sts-permission read_write
          -v