gatk-tests.yml

name: Build and Test GATK 4
on:
  push:
    branches:
      - 'master'
  pull_request:
  workflow_dispatch:

env:
  CROMWELL_VERSION: 84
  TERM: dumb
  GRADLE_OPTS: "-Xmx2048m -Dorg.gradle.daemon=false"
  HELLBENDER_TEST_INPUTS: gs://hellbender/test/resources/
  HELLBENDER_TEST_STAGING: gs://hellbender-test-logs/staging/
  HELLBENDER_TEST_LOGS: /hellbender-test-logs/build_reports/
  HELLBENDER_TEST_PROJECT: broad-dsde-dev
  TEST_VERBOSITY: minimal
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  ## This workaround is necessary since there is no equivalent to the old TRAVIS_SECURE_ENVIRONMENT variable that indicated
  ## if a run was privileged and had secrets. Since the GCP credentials are necessary for all tests in order to upload their,
  ## results that makes them a reasonable proxy for testing the credentials of this entire execution. https://github.com/actions/runner/issues/520
  check-secrets:
    name: check if the environment has privileges
    outputs:
      google-credentials: ${{ steps.google-credentials.outputs.defined }}
    runs-on: ubuntu-latest
    steps:
      - id: google-credentials
        env:
          GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}
        if: "${{ env.GCP_CREDENTIALS != '' }}"
        run: echo defined=true >> $GITHUB_OUTPUT


  buildDocker:
    name: build and push gatk docker to artifacts
    runs-on: ubuntu-latest
    steps:
      - name: checkout gatk
        uses: actions/checkout@v3

      - uses: ./.github/actions/purge-runner-disk

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=raw,value=${{github.sha}}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Build and push Docker image
        uses: docker/build-push-action@v4
        with:
          context: .
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          outputs: type=docker,dest=/tmp/myimage.tar
      # By uploading the docker image as an artifact we save ourselves the requirement for the image to be built with ghcr push permission
      - name: Upload docker image as artifact 
        uses: actions/upload-artifact@v3
        with:
          name: gatkDockerImage
          path: /tmp/myimage.tar

  #Run our non-docker tests
  test:
    runs-on: ubuntu-latest
    needs: check-secrets
    strategy:
      matrix:
        java: [ 17.0.6+10 ]
        experimental: [ false ]
        scalaVersion: [ 2.13 ]
        testType: [ cloud, integration, unit ]
      fail-fast: false
    continue-on-error: ${{ matrix.experimental }}
    env:
      TEST_TYPE: ${{ matrix.testType }}
      SCALA_VERSION: ${{ matrix.scalaVersion }}
    name: Java ${{ matrix.Java }} build and test ${{ matrix.testType }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: ./.github/actions/purge-runner-disk
      - name: 'Set up java ${{ matrix.Java }}'
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.Java }}
          distribution: 'temurin'
          cache: gradle

      - name: 'Compile with Gradle'
        run: |
          ./gradlew compileJava ; ./gradlew installDist

      #Google Cloud stuff
      - id: 'gcloud-auth'
        if: needs.check-secrets.outputs.google-credentials == 'true'
        uses: google-github-actions/auth@v0
        with:
          credentials_json: ${{ secrets.GCP_CREDENTIALS }}
          project_id: ${{ env.HELLBENDER_TEST_PROJECT }}
          create_credentials_file: true

      - name: "export the credentials for GATK tests"
        if: needs.check-secrets.outputs.google-credentials == 'true'
        run: echo "HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV

      - name: 'Set up Cloud SDK'
        if: needs.check-secrets.outputs.google-credentials == 'true'
        uses: google-github-actions/setup-gcloud@v0

      - name: pull lfs files
        run: git lfs pull

      - name: compile test code
        run: ./gradlew compileTestJava

      - name: run-tests
        if: ${{ needs.check-secrets.outputs.google-credentials == 'true' || matrix.testType != 'cloud'}}
        id: jacoco-tests
        run: |
          ./gradlew --daemon -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport

      - uses: ./.github/actions/upload-gatk-test-results
        if: always()
        with:
          warnPR: ${{ github.event_name == 'pull_request' && steps.jacoco-tests.outcome != 'success' }}
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          job-matrix-id: ${{ github.run_id }}.1${{ strategy.job-index }}
          repo-path: ${{ github.ref_name }}_${{ github.run_id }}.1${{ strategy.job-index }}
          bot-comment-key: ${{ secrets.GATK_BOT_COMMENT_KEY }}
          identifier: Java ${{ matrix.Java }} build and test ${{ matrix.testType }}
          only-artifact: ${{ needs.check-secrets.outputs.google-credentials != 'true' }}


  #Run our docker tests
  testOnDocker:
    needs: [buildDocker, check-secrets]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        java: [ 17.0.6+10 ]
        experimental: [ false ]
        scalaVersion: [ 2.13 ]
        testType: [ integration, unit, variantcalling, conda ]
      fail-fast: false
    continue-on-error: ${{ matrix.experimental }}
    env:
      TEST_TYPE: ${{ matrix.testType }}
      SCALA_VERSION: ${{ matrix.scalaVersion }}

    name: Java ${{ matrix.Java }} test on docker ${{matrix.testType}}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 #TODO make this shallow

      - uses: ./.github/actions/purge-runner-disk

      - name: 'Set up java ${{ matrix.Java }}'
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.Java }}
          distribution: 'temurin'
          cache: gradle

      #Google Cloud stuff
      - id: 'gcloud-auth'
        uses: google-github-actions/auth@v0
        if: needs.check-secrets.outputs.google-credentials == 'true'
        with:
          credentials_json: ${{ secrets.GCP_CREDENTIALS }}
          project_id: ${{ env.HELLBENDER_TEST_PROJECT }}
          create_credentials_file: true

      - name: "export the credentials for GATK tests"
        if: needs.check-secrets.outputs.google-credentials == 'true'
        run: echo "HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV

      - name: 'Set up Cloud SDK'
        if: needs.check-secrets.outputs.google-credentials == 'true'
        uses: google-github-actions/setup-gcloud@v0

      - name: build test jars
        run: ./gradlew clean shadowTestClassJar shadowTestJar

      - name: set up test folders and links
        run: |
          mkdir ./testJars;
          echo $( find ./build/libs/ -name "gatk*test.jar");
          cp -r $( find ./build/libs/ -name "gatk*test.jar" ) ./testJars;
          cp -r $( find ./build/libs/ -name "gatk*testDependencies.jar" ) ./testJars;
          mkdir -p build/reports/;
          chmod -R a+w build/reports/;
          cp scripts/docker/dockertest.gradle .

      - name: Download docker image artifact
        uses: actions/download-artifact@v3
        with:
          name: gatkDockerImage
          path: /tmp
      - name: Load Docker image
        run: |
          docker load --input /tmp/myimage.tar
          docker image ls -a

      - name: pull lfs files
        run: git lfs pull

      - name: run-docker-tests
        id: docker-jacocco-tests
        run: |
          docker run \
            -v $(pwd):/gatkCloneMountPoint:cached \
            -v $(pwd)/testJars:/jars:cached \
            --rm \
            -e "scala.version=${SCALA_VERSION}" \
            -e "TEST_VERBOSITY=minimal" \
            -e "TEST_TYPE=${TEST_TYPE}" \
            -t  ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} \
            bash --init-file /gatk/gatkenv.rc /root/run_unit_tests.sh;
          TEST_EXIT_VALUE=$?;
          $( exit ${TEST_EXIT_VALUE} );
          sudo chmod -R a+w build/reports/;
          mkdir build/reports/tests/test \
           && cp -rp build/reports/tests/testOnPackagedReleaseJar/* build/reports/tests/test \
           && rm -r build/reports/tests/testOnPackagedReleaseJar;

      - uses: ./.github/actions/upload-gatk-test-results
        if: always()
        with:
          warnPR: ${{ github.event_name == 'pull_request' && steps.docker-jacocco-tests.outcome != 'success' }}
          is-docker: true
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          job-matrix-id: ${{ github.run_id }}.${{ strategy.job-index }}
          repo-path: ${{ github.ref_name }}_${{ github.run_id }}.${{ strategy.job-index }}
          bot-comment-key: ${{ secrets.GATK_BOT_COMMENT_KEY }}
          identifier: Java ${{ matrix.Java }} test on docker ${{matrix.testType}}
          only-artifact: ${{ needs.check-secrets.outputs.google-credentials != 'true' }}

  #run wdl validation on WDLs in the scripts directory
  scriptsWdlValidation:
    runs-on: ubuntu-latest
    name: Validate script WDLs using womtools
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: ./.github/actions/purge-runner-disk
      - name: Set up java 17
        uses: actions/setup-java@v3
        with:
          java-version: '17'
          distribution: 'temurin'
          cache: gradle
      - uses: ./.github/actions/install-cromwell
        with:
          CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }}
      - name: Run Scripts WDL Validation Test
        run: ./gradlew gatkValidateScriptsWdl

  #run wdl validation on generated WDLs
  generatedWdlValidation:
    runs-on: ubuntu-latest
    name: Validate generated WDLs using womtools
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: ./.github/actions/purge-runner-disk
      - name: Set up java 17
        uses: actions/setup-java@v3
        with:
          java-version: '17'
          distribution: 'temurin'
          cache: gradle
      - uses: ./.github/actions/install-cromwell
        with:
          CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }}

      - name: Run Generated WDL Validation Test
        run: ./gradlew gatkValidateGeneratedWdl

  #Run our various targeted medium scale wdl wiring tests
  wdlTests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        wdlTest: [ 'RUN_CNV_GERMLINE_COHORT_WDL', 'RUN_CNV_GERMLINE_CASE_WDL', 'RUN_CNV_SOMATIC_WDL', 'RUN_M2_WDL', 'RUN_CNN_WDL', 'RUN_VCF_SITE_LEVEL_FILTERING_WDL' ]
    continue-on-error: true
    name: WDL test ${{ matrix.wdlTest }} on cromwell
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: ./.github/actions/purge-runner-disk
      - name: pull lfs files
        run: git lfs pull
      - name: Set up java 17
        uses: actions/setup-java@v3
        with:
          java-version: '17'
          distribution: 'temurin'
          cache: gradle
      - uses: ./.github/actions/install-cromwell
        with:
          CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }}

      - name: "Exporting Environment Variables for Wdl Scripts"
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]
          then
            echo "CI_PULL_REQUEST=${{ github.event.number }}" >> $GITHUB_ENV 
          fi
          echo "CI_COMMIT=${{ github.sha }}" >> $GITHUB_ENV

      - name: "CNV_GERMLINE_COHORT_WDL_TEST"
        if: ${{ matrix.wdlTest  == 'RUN_CNV_GERMLINE_COHORT_WDL' }}
        run: |
          echo "Running CNV germline cohort workflow";
          bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh COHORT;

      - name: "CNV_GERMLINE_CASE_WDL_TEST"
        if: ${{ matrix.wdlTest == 'RUN_CNV_GERMLINE_CASE_WDL' }}
        run: |
          echo "Running CNV germline case workflow";
          bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh CASE;

      - name: "CNV_SOMATIC_WDL_TEST"
        if: ${{ matrix.wdlTest == 'RUN_CNV_SOMATIC_WDL' }}
        run: |
          echo "Running CNV somatic workflows";
          bash scripts/cnv_cromwell_tests/somatic/run_cnv_somatic_workflows.sh;

      - name: "M2_WDL_TEST"
        if: ${{ matrix.wdlTest == 'RUN_M2_WDL' }}
        run: |
          echo "Deleting some unused files before running M2 WDL...";
          rm -Rf src/test/resources/large/VQSR;
          echo "Running M2 WDL";
          bash scripts/m2_cromwell_tests/run_m2_wdl.sh;

      - name: "CNN_WDL_TEST"
        if: ${{ matrix.wdlTest == 'RUN_CNN_WDL' }}
        run: |
          echo "Running CNN WDL";
          bash scripts/cnn_variant_cromwell_tests/run_cnn_variant_wdl.sh;

      - name: "VCF_SITE_LEVEL_FILTERING_WDL_TEST"
        if: ${{ matrix.wdlTest == 'RUN_VCF_SITE_LEVEL_FILTERING_WDL' }}
        run: |
          echo "Running VCF Site Level Filtering WDL";
          bash scripts/vcf_site_level_filtering_cromwell_tests/run_vcf_site_level_filtering_wdl.sh;