ci.yml

# Copyright 2023 The Authors (see AUTHORS file)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: 'ci'

on:
  push:
    branches:
      - 'main'
  pull_request:
    branches:
      - 'main'
  workflow_dispatch:
  workflow_call:
  merge_group:
    types: ['checks_requested']

env:
  DOCKER_REGISTRY: 'us-docker.pkg.dev'
  DOCKER_TAG: '${{ github.sha }}'
  DOCKER_REPO: 'us-docker.pkg.dev/lumberjack-ci-images-i-1f/ci-images'
  WIF_PROVIDER: 'projects/567170510917/locations/global/workloadIdentityPools/github-automation/providers/lumberjack-ci-i'
  WIF_SERVICE_ACCOUNT: 'github-automation-bot@gha-lumberjack-ci-i-9d0848.iam.gserviceaccount.com'
  SERVER_PROJECT_ID: 'lumberjack-server-i-8eb3ef'
  CLIENT_PROJECT_ID: 'lumberjack-app-i-84af7b'
  SERVER_RUN_SA: 'server-run-22ad10@lumberjack-server-i-8eb3ef.iam.gserviceaccount.com'
  CLIENT_RUN_SA: 'client-run-22ad10@lumberjack-app-i-84af7b.iam.gserviceaccount.com'
  BIGQUERY_DATASET_ID: 'audit_logs.audit_abcxyz_data_access'

concurrency:
  group: '${{ github.workflow }}-${{ github.head_ref || github.ref }}-ci'
  cancel-in-progress: true

jobs:
  # Linting jobs - terraform, java, go
  terraform_lint:
    uses: 'abcxyz/pkg/.github/workflows/terraform-lint.yml@main' # ratchet:exclude
    with:
      directory: 'terraform'
      terraform_version: '1.2'

  yaml_lint:
    uses: 'abcxyz/pkg/.github/workflows/yaml-lint.yml@main' # ratchet:exclude

  go_lint:
    uses: 'abcxyz/pkg/.github/workflows/go-lint.yml@main' # ratchet:exclude

  java_lint:
    uses: 'abcxyz/pkg/.github/workflows/java-lint.yml@main' # ratchet:exclude
    with:
      directory: 'clients/java-logger'
      java_version: '11'

  # Unit tests - java, go
  go_test:
    uses: 'abcxyz/pkg/.github/workflows/go-test.yml@main' # ratchet:exclude

  java_test:
    runs-on: 'ubuntu-latest'
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - uses: 'actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11' # ratchet:actions/checkout@v4

      - uses: 'actions/setup-java@9704b39bf258b59bc04b50fa2dd55e9ed76b47a8' # ratchet:actions/setup-java@v4
        with:
          java-version: '11'
          distribution: 'temurin'

      # We don't actually need to authenticate here, but the Artifact Regsitry
      # Wagon[1] expects authentication and infinitely retries if it does not
      # find any.
      #
      # [1]: https://github.com/GoogleCloudPlatform/artifact-registry-maven-tools
      - uses: 'google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c' # ratchet:google-github-actions/auth@v2
        with:
          workload_identity_provider: '${{ env.WIF_PROVIDER }}'
          service_account: '${{ env.WIF_SERVICE_ACCOUNT }}'

          # The Artifact Registry maven wagon looks for Google Application Default Credentials.
          # https://github.com/GoogleCloudPlatform/artifact-registry-maven-tools
      - name: 'Run tests'
        run: |-
          mvn --no-transfer-progress --batch-mode clean test -f clients/java-logger

  # lint_and_unit is a virtual job that is used as dependencies for later jobs.
  lint_and_unit:
    runs-on: 'ubuntu-latest'
    needs:
      - 'terraform_lint'
      - 'yaml_lint'
      - 'go_lint'
      - 'java_lint'
      - 'go_test'
      - 'java_test'
    steps:
      - run: 'echo prechecks complete'

  # Build the main lumberjack server and push to artifact registry
  build-lumberjack-server:
    runs-on: 'ubuntu-latest'
    needs:
      - 'lint_and_unit'
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - uses: 'docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3' # ratchet:docker/setup-qemu-action@v3

      - uses: 'actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11' # ratchet:actions/checkout@v4

      - uses: 'actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491' # ratchet:actions/setup-go@v5
        with:
          go-version-file: 'go.mod'

      - id: 'auth'
        uses: 'google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c' # ratchet:google-github-actions/auth@v2
        with:
          workload_identity_provider: '${{ env.WIF_PROVIDER }}'
          service_account: '${{ env.WIF_SERVICE_ACCOUNT }}'
          token_format: 'access_token'

      - uses: 'docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d' # ratchet:docker/login-action@v3
        with:
          username: 'oauth2accesstoken'
          password: '${{ steps.auth.outputs.access_token }}'
          registry: '${{ env.DOCKER_REGISTRY }}'

      # goreleaser requires a tag to publish images to container registry.
      # We create a local tag to make it happy.
      - run: |-
          git config user.name "${GITHUB_ACTOR}"
          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
          git tag -f -a v0.0.0-ci -m "CI run"

      - name: 'Build the main server container and push to the registry with goreleaser'
        uses: 'goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8' # ratchet:goreleaser/goreleaser-action@v5
        with:
          version: 'v1.16.1' # Manually pinned
          args: 'release -f .goreleaser.docker.yaml --clean --skip-validate'

  # Build the 4 integration test client variants, go-grpc, go-http, java-grpc and java-http
  build-go-grpc-client:
    runs-on: 'ubuntu-latest'
    needs:
      - 'lint_and_unit'
    permissions:
      contents: 'read'
      id-token: 'write'
    env:
      prefix: 'ljc'
      lang: 'go'
      type: 'grpc'
      file: './clients/go/test/grpc-app/Dockerfile'
      run_from: './'
    steps:
      - uses: 'actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11' # ratchet:actions/checkout@v4

      - id: 'auth'
        uses: 'google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c' # ratchet:google-github-actions/auth@v2
        with:
          workload_identity_provider: '${{ env.WIF_PROVIDER }}'
          service_account: '${{ env.WIF_SERVICE_ACCOUNT }}'
          token_format: 'access_token'

      - uses: 'docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d' # ratchet:docker/login-action@v3
        with:
          username: 'oauth2accesstoken'
          password: '${{ steps.auth.outputs.access_token }}'
          registry: '${{ env.DOCKER_REGISTRY }}'

      - name: 'Build the integration test server container and push to the registry'
        uses: 'docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56' # ratchet:docker/build-push-action@v5
        with:
          push: true
          tags: '${{ env.DOCKER_REPO }}/${{env.prefix}}-${{env.lang}}-${{env.type}}:${{env.DOCKER_TAG}}'
          file: '${{ env.file }}'
          context: '${{ env.run_from }}'

  build-go-http-client:
    runs-on: 'ubuntu-latest'
    needs:
      - 'lint_and_unit'
    permissions:
      contents: 'read'
      id-token: 'write'
    env:
      prefix: 'ljc'
      lang: 'go'
      type: 'http'
      file: './clients/go/test/shell/Dockerfile'
      run_from: './'
    steps:
      - uses: 'actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11' # ratchet:actions/checkout@v4

      - id: 'auth'
        uses: 'google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c' # ratchet:google-github-actions/auth@v2
        with:
          workload_identity_provider: '${{ env.WIF_PROVIDER }}'
          service_account: '${{ env.WIF_SERVICE_ACCOUNT }}'
          token_format: 'access_token'

      - uses: 'docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d' # ratchet:docker/login-action@v3
        with:
          username: 'oauth2accesstoken'
          password: '${{ steps.auth.outputs.access_token }}'
          registry: '${{ env.DOCKER_REGISTRY }}'

      - name: 'Build the integration test server container and push to the registry'
        uses: 'docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56' # ratchet:docker/build-push-action@v5
        with:
          push: true
          tags: '${{ env.DOCKER_REPO }}/${{env.prefix}}-${{env.lang}}-${{env.type}}:${{env.DOCKER_TAG}}'
          file: '${{ env.file }}'
          context: '${{ env.run_from }}'

  build-java-grpc-client:
    runs-on: 'ubuntu-latest'
    needs:
      - 'lint_and_unit'
    permissions:
      contents: 'read'
      id-token: 'write'
    env:
      prefix: 'ljc'
      lang: 'java'
      type: 'grpc'
      file: './clients/java-logger/scripts/server_app.dockerfile'
      run_from: './'
    steps:
      - uses: 'actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11' # ratchet:actions/checkout@v4

      - id: 'auth'
        uses: 'google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c' # ratchet:google-github-actions/auth@v2
        with:
          workload_identity_provider: '${{ env.WIF_PROVIDER }}'
          service_account: '${{ env.WIF_SERVICE_ACCOUNT }}'
          token_format: 'access_token'

      - uses: 'docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d' # ratchet:docker/login-action@v3
        with:
          username: 'oauth2accesstoken'
          password: '${{ steps.auth.outputs.access_token }}'
          registry: '${{ env.DOCKER_REGISTRY }}'

      - name: 'Build the integration test server container and push to the registry'
        uses: 'docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56' # ratchet:docker/build-push-action@v5
        with:
          push: true
          tags: '${{ env.DOCKER_REPO }}/${{env.prefix}}-${{env.lang}}-${{env.type}}:${{env.DOCKER_TAG}}'
          file: '${{ env.file }}'
          context: '${{ env.run_from }}'

  build-java-http-client:
    runs-on: 'ubuntu-latest'
    needs:
      - 'lint_and_unit'
    permissions:
      contents: 'read'
      id-token: 'write'
    env:
      prefix: 'ljc'
      lang: 'java'
      type: 'http'
      file: './clients/java-logger/scripts/shell_app.dockerfile'
      run_from: './'
    steps:
      - uses: 'actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11' # ratchet:actions/checkout@v4

      - id: 'auth'
        uses: 'google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c' # ratchet:google-github-actions/auth@v2
        with:
          workload_identity_provider: '${{ env.WIF_PROVIDER }}'
          service_account: '${{ env.WIF_SERVICE_ACCOUNT }}'
          token_format: 'access_token'

      - uses: 'docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d' # ratchet:docker/login-action@v3
        with:
          username: 'oauth2accesstoken'
          password: '${{ steps.auth.outputs.access_token }}'
          registry: '${{ env.DOCKER_REGISTRY }}'

      - name: 'Build the integration test server container and push to the registry'
        uses: 'docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56' # ratchet:docker/build-push-action@v5
        with:
          push: true
          tags: '${{ env.DOCKER_REPO }}/${{env.prefix}}-${{env.lang}}-${{env.type}}:${{env.DOCKER_TAG}}'
          file: '${{ env.file }}'
          context: '${{ env.run_from }}'

  # build is a virtual job that is used as dependencies for later jobs.
  build:
    runs-on: 'ubuntu-latest'
    needs:
      - 'build-lumberjack-server'
      - 'build-go-grpc-client'
      - 'build-go-http-client'
      - 'build-java-grpc-client'
      - 'build-java-http-client'
    steps:
      - run: 'echo build complete'

  # Deploy integration test infrastructure and run integration tests
  integration:
    runs-on: 'ubuntu-latest'
    needs:
      - 'build'
    permissions:
      contents: 'read'
      id-token: 'write'
    env:
      tf_module_dir: './terraform/modules/ci'
    steps:
      - uses: 'actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11' # ratchet:actions/checkout@v4

      - id: 'auth'
        uses: 'google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c' # ratchet:google-github-actions/auth@v2
        with:
          workload_identity_provider: '${{ env.WIF_PROVIDER }}'
          service_account: '${{ env.WIF_SERVICE_ACCOUNT }}'
          token_format: 'access_token'

      # TODO: Workload Identity doesn't work with GCS Provider.
      # https://github.com/google-github-actions/auth/issues/89
      - name: 'Workaround Terraform issue'
        shell: 'bash'
        run: |-
          echo "GOOGLE_OAUTH_ACCESS_TOKEN=${{ steps.auth.outputs.access_token }}" >> $GITHUB_ENV

      - uses: 'hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36' # ratchet:hashicorp/setup-terraform@v3
        with:
          terraform_wrapper: false

      - name: 'Write tfvars file'
        env:
          client_images: '\{\"ljc-go-grpc\":\"${{env.DOCKER_REPO}}/ljc-go-grpc:${{env.DOCKER_TAG}}\",\"ljc-go-http\":\"${{env.DOCKER_REPO}}/ljc-go-http:${{env.DOCKER_TAG}}\",\"ljc-java-grpc\":\"${{env.DOCKER_REPO}}/ljc-java-grpc:${{env.DOCKER_TAG}}\",\"ljc-java-http\":\"${{env.DOCKER_REPO}}/ljc-java-http:${{env.DOCKER_TAG}}\"\}'
          server_image: '${{env.DOCKER_REPO}}/lumberjack-server:${{env.DOCKER_TAG}}'
        run: |-
          var_file=/tmp/ci.tfvars

          touch /tmp/ci.tfvars
          echo commit_sha=\"${{ env.DOCKER_TAG }}\" >> ${var_file};
          echo server_project_id=\"${{ env.SERVER_PROJECT_ID }}\" >> ${var_file};
          echo server_image=\"${{ env.server_image }}\" >> ${var_file};
          echo server_run_sa=\"${{ env.SERVER_RUN_SA }}\" >> ${var_file};
          echo server_service_name=\"lumberjack-server\" >> ${var_file};
          echo client_project_id=\"${{ env.CLIENT_PROJECT_ID }}\" >> ${var_file};
          echo client_run_sa=\"${{ env.CLIENT_RUN_SA }}\" >> ${var_file};
          echo client_images=${{ env.client_images }} >> ${var_file};
          echo >> ${var_file};

          cat ${var_file};
          echo "tfvar_file=${var_file}" >> $GITHUB_ENV;

      - name: 'Init the terraform infrastructure'
        run: 'terraform -chdir=${{ env.tf_module_dir }} init'

      - name: 'Apply the infrastructure updates'
        id: 'tf_apply'
        run: |-
          terraform -chdir=${{ env.tf_module_dir}} apply -auto-approve -var-file=${{env.tfvar_file}};

          terraform -chdir=${{ env.tf_module_dir}} output -json ingestion_backend_client_endpoints | jq -c > ingestion_backend_client_output.json;

          terraform -chdir=${{ env.tf_module_dir}} output -json cloudlogging_backend_client_endpoints | jq -c > cloudlogging_backend_client_output.json;

          ingestion_backend_grpc_client_endpoints=$(jq -c '{"ljc-go-grpc","ljc-java-grpc"} | to_entries | map(.value|tostring)' < ingestion_backend_client_output.json);
          ingestion_backend_http_client_endpoints=$(jq -c '{"ljc-go-http","ljc-java-http"} | to_entries | map(.value|tostring)' < ingestion_backend_client_output.json);

          cloudlogging_backend_grpc_client_endpoints=$(jq -c '{"ljc-go-grpc","ljc-java-grpc"} | to_entries | map(.value|tostring)' < cloudlogging_backend_client_output.json);
          cloudlogging_backend_http_client_endpoints=$(jq -c '{"ljc-go-http","ljc-java-http"} | to_entries | map(.value|tostring)' < cloudlogging_backend_client_output.json);

          grpc_client_endpoints=$(echo -e "${ingestion_backend_grpc_client_endpoints}${cloudlogging_backend_grpc_client_endpoints}" | jq -cs 'flatten(1)')
          http_client_endpoints=$(echo -e "${ingestion_backend_http_client_endpoints}${cloudlogging_backend_http_client_endpoints}" | jq -cs 'flatten(1)')

          echo "GRPC_ENDPOINTS=${grpc_client_endpoints}" >> $GITHUB_ENV;
          echo "HTTP_ENDPOINTS=${http_client_endpoints}" >> $GITHUB_ENV;

      - uses: 'actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491' # ratchet:actions/setup-go@v5
        with:
          go-version-file: 'go.mod'

      - name: 'Run tests'
        env:
          TEST_INTEGRATION: 'true'
          AUDIT_CLIENT_TEST_JUSTIFICATION_SUB: '${{ env.WIF_SERVICE_ACCOUNT }}'
          AUDIT_CLIENT_TEST_PROJECT_ID: '${{ env.SERVER_PROJECT_ID }}'
          AUDIT_CLIENT_TEST_BIGQUERY_DATASET: '${{ env.BIGQUERY_DATASET_ID }}'
          AUDIT_CLIENT_TEST_PRIVATE_KEY_PATH: '${{ github.workspace }}/integration/testrunner/test_private_key'
        run: |-
          export AUDIT_CLIENT_TEST_IDTOKEN="$(gcloud auth print-identity-token --impersonate-service-account=${{ env.WIF_SERVICE_ACCOUNT }} --include-email)"
          go test ./integration/testrunner -timeout=20m

      - name: 'Cleanup the infrastructure'
        if: '${{ always() }}'
        run: |-
          terraform -chdir=${{ env.tf_module_dir}} apply -destroy -auto-approve -var-file=${{env.tfvar_file}};