azure-pipelines.yml

# This pipeline has four main stages (with two extra ones for optimization reasons):
# 1) Building docker images:
#     - anonlink-nginx
#     - anonling-app
#     - anonlink-benchmark (done in a separate stage 1bis) to be able to depend on it)
#     - anonlink-docs-tutorials (done in a separate stage 1ter) to be able to depend on it)
# 2) Running the integration tests using docker-compose
# 3) Running the benchmark using docker-compose
# 4) Running the tests for the tutorials using docker-compose
# 5) Deploying on Kubernetes and running the corresponding tests.
# The dependencies are the following:
# 1 -> 2 -> 5
# 1, 1bis -> 3
# 1, 1ter -> 4
#
# The dockerhub login is a secret in Azure Pipeline.
# The kubeconfig file is a secret file in Azure Pipeline.
#
# The resources used by this pipeline are available in the folder `.azurePipeline`,
# where there are some kubectl templates, azure pipeline step templates and bash scripts.

variables:
  backendImageName: data61/anonlink-app
  testE2EImageName: data61/anonlink-test
  frontendImageName: data61/anonlink-nginx
  tutorialImageName: data61/anonlink-docs-tutorials
  benchmarkingImageName: data61/anonlink-benchmark

  # The Kubernetes Service Connection to use - must be configured in Azure DevOps.
  CLUSTER: 'k8s'

# This pipeline should be triggered by every push to any branches, but should not be used
# for external PRs.
pr: none
trigger:
  branches:
    include:
    - '*'

stages:
- stage: stage_base_docker_image_build
  displayName: Base Docker build
  dependsOn: []
  jobs:
  - job: HashBaseDependencies
    displayName: Hash Dependencies
    pool:
      vmImage: 'ubuntu-latest'
    steps:
      - template: .azurePipeline/templateSetVariableDockerBaseTag.yml
      - script: |
          # Determine if we need to build the base by checking if docker hub already
          # has the image?
          echo "Trying to pull $(SetDockerBaseTag.DOCKER_BASE_TAG)"
          if docker pull data61/anonlink-base:$(SetDockerBaseTag.DOCKER_BASE_TAG) ; then
              echo "Base image pulled. Skipping building the base image."
              echo "##vso[task.setvariable variable=BUILD_BASE;isOutput=true]false"
          else
              echo "Base image with correct tag wasn't present."
              echo "##vso[task.setvariable variable=BUILD_BASE;isOutput=true]true"
          fi
        name: BaseRequired

  - template: .azurePipeline/templateDockerBuildPush.yml
    parameters:
      folder: './base'
      dependsOn: HashBaseDependencies
      imageName: data61/anonlink-base
      jobName: 'anonlink_base'
      imageTag: $[dependencies.HashBaseDependencies.outputs['SetDockerBaseTag.DOCKER_BASE_TAG']]
      skip: $[eq(dependencies.HashBaseDependencies.outputs['BaseRequired.BUILD_BASE'], 'false')]

- stage: stage_docker_image_build
  displayName: Anonlink Docker build
  dependsOn: [stage_base_docker_image_build]
  jobs:
  # Why do we recompute the base hash? Because we can't pass variables between stages.
  # https://github.com/microsoft/azure-pipelines-tasks/issues/4743
  - job: HashBaseDependencies
    displayName: Hash Dependencies
    pool:
      vmImage: 'ubuntu-latest'
    steps:
      - template: .azurePipeline/templateSetVariableDockerBaseTag.yml
  - template: .azurePipeline/templateDockerBuildPush.yml
    parameters:
      folder: './backend'
      jobName: 'anonlink_app'
      dependsOn: HashBaseDependencies
      imageName: data61/anonlink-app
      dockerBuildVersion: "$[dependencies.HashBaseDependencies.outputs['SetDockerBaseTag.DOCKER_BASE_TAG']]"

- stage: stage_docker_e2e_test_image_build
  displayName: Build E2E Test Docker image
  dependsOn: [stage_base_docker_image_build]
  jobs:
  # Why do we recompute the base hash? Because we can't pass variables between stages.
  # https://github.com/microsoft/azure-pipelines-tasks/issues/4743
  - job: HashBaseDependencies
    displayName: Hash Dependencies
    pool:
      vmImage: 'ubuntu-latest'
    steps:
      - template: .azurePipeline/templateSetVariableDockerBaseTag.yml
  - template: .azurePipeline/templateDockerBuildPush.yml
    parameters:
      folder: './e2etests'
      jobName: 'anonlink_e2e_test'
      dependsOn: HashBaseDependencies
      imageName: data61/anonlink-test
      dockerBuildVersion: "$[dependencies.HashBaseDependencies.outputs['SetDockerBaseTag.DOCKER_BASE_TAG']]"

- stage: stage_docker_nginx_image_build
  displayName: Nginx Docker build
  dependsOn: []
  jobs:
  - template: .azurePipeline/templateDockerBuildPush.yml
    parameters:
      folder: '.'
      dockerFilePath: './frontend/Dockerfile'
      imageName: data61/anonlink-nginx
      jobName: 'anonlink_nginx'

- stage: stage_benchmark_image_build
  displayName: Build benchmark image
  dependsOn: []
  jobs:
  - template: .azurePipeline/templateDockerBuildPush.yml
    parameters:
      folder: './benchmarking'
      imageName: data61/anonlink-benchmark
      jobName: 'anonlink_benchmark'

- stage: stage_docs_tutorial_image_build
  displayName: Build docs tutorials image
  dependsOn: []
  jobs:
  - template: .azurePipeline/templateDockerBuildPush.yml
    parameters:
      folder: './docs/tutorial'
      imageName: data61/anonlink-docs-tutorials
      jobName: 'anonlink_docs_tutorials'

- stage: stage_integration_tests
  displayName: Integration Tests
  dependsOn:
  - stage_docker_image_build
  jobs:
  - job: Integration
    timeoutInMinutes: 15
    variables:
      resultFile: testResults.xml
    displayName: Integration Tests
    pool:
      vmImage: 'ubuntu-latest'
    steps:
    - template: .azurePipeline/templateSetVariableDockerTag.yml
    - script: |
        ./.azurePipeline/runDockerComposeTests.sh --no-ansi -p es$(DOCKER_TAG)$(Build.SourceVersion) -t $(DOCKER_TAG) -o $(resultFile) --type integrationtests
      displayName: 'Start docker compose'
    - task: PublishTestResults@2
      condition: succeededOrFailed()
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(resultFile)'
        testRunTitle: 'Publish integration test results'
        failTaskOnFailedTests: true

- stage: stage_benchmark
  displayName: Benchmark
  dependsOn:
  - stage_docker_image_build
  - stage_benchmark_image_build
  jobs:
  - job: Benchmark
    timeoutInMinutes: 15
    variables:
      resultFile: results.json
    displayName: Benchmark
    pool:
      vmImage: 'ubuntu-latest'
    steps:
    - template: .azurePipeline/templateSetVariableDockerTag.yml
    - script: |
        ./.azurePipeline/runDockerComposeTests.sh --no-ansi -p es$(DOCKER_TAG)$(Build.SourceVersion) -t $(DOCKER_TAG) -o $(resultFile) --type benchmark
      displayName: 'Start docker compose benchmark'
    # Publish Pipeline Artifact
    # Publish a local directory or file as a named artifact for the current pipeline.
    - task: PublishPipelineArtifact@0
      inputs:
        artifactName: 'benchmark'
        targetPath: $(resultFile)

- stage: stage_tutorials_tests
  displayName: Tutorials tests
  dependsOn:
  - stage_docker_image_build
  - stage_docs_tutorial_image_build
  jobs:
  - job: TutorialsTests
    timeoutInMinutes: 10
    variables:
      resultFile: tutorialTestResults.xml
    displayName: Tutorials Tests
    pool:
      vmImage: 'ubuntu-latest'
    steps:
    - template: .azurePipeline/templateSetVariableDockerTag.yml
    - script: |
        ./.azurePipeline/runDockerComposeTests.sh --no-ansi -p es$(DOCKER_TAG)$(Build.SourceVersion) -t $(DOCKER_TAG) -o $(resultFile) --type tutorials
      displayName: 'Test notebook tutorials'
    - task: PublishTestResults@2
      condition: succeededOrFailed()
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(resultFile)'
        testRunTitle: 'Publish tutorials tests results'
        failTaskOnFailedTests: true

- stage: stage_k8s_deployment
  displayName: Kubernetes deployment
  dependsOn: [stage_docker_image_build]
  jobs:
  - job: job_k8s_deployment
    displayName: Kubernetes deployment and test
    timeoutInMinutes: 40

    variables:
      # All the deployments for tests are done in the namespace `test-azure`, simplifying
      # debugging or cleaning if something wrong happens.
      NAMESPACE: test-azure

    pool:
      vmImage: 'ubuntu-latest'
    steps:
      # Prepare the variables required during the scripts.
      - template: .azurePipeline/templateSetVariableDockerTag.yml
      - template: .azurePipeline/templateSetVariableReleaseName.yml
      - script: |
          echo "##vso[task.setvariable variable=PVC]$(DEPLOYMENT)-test-results"
          echo "##vso[task.setvariable variable=SERVICE]$(DEPLOYMENT)-entity-service-api"
          echo $(backendImageName):$(DOCKER_TAG) | xargs -I@ echo "##vso[task.setvariable variable=IMAGE_NAME_WITH_TAG]@"
          echo $(testE2EImageName):$(DOCKER_TAG) | xargs -I@ echo "##vso[task.setvariable variable=TEST_E2E_IMAGE_NAME_WITH_TAG]@"
          echo $(DEPLOYMENT)-tmppod | xargs -I@ echo "##vso[task.setvariable variable=POD_NAME]@"
        displayName: 'Set variables for service, test result volume and pod'
      - task: Kubernetes@1
        displayName: 'Create a new namespace'
        inputs:
          connectionType: 'Kubernetes Service Connection'
          kubernetesServiceEndpoint: $(CLUSTER)
          command: apply
          useConfigurationFile: true
          inline: '{ "kind": "Namespace", "apiVersion": "v1", "metadata": { "name": "$(NAMESPACE)" }}'

      # The file `.azurePipeline/k8s_test_pvc.yaml.tmpl` is a template for the creation of persistent volume via kubectl
      - script: |
          cat .azurePipeline/k8s_test_pvc.yaml.tmpl | \
          sed 's|\$PVC'"|$(PVC)|g" | \
          sed 's|\$DEPLOYMENT_NAME'"|$(DEPLOYMENT)|g" > $(Build.ArtifactStagingDirectory)/k8s_test_pvc.yaml
        displayName: 'Create test result volume claim from template'

      - task: KubernetesManifest@0
        displayName: 'Create test result volume claim'
        inputs:
          action: 'deploy'
          kubernetesServiceConnection: '$(CLUSTER)'
          manifests: '$(Build.ArtifactStagingDirectory)/k8s_test_pvc.yaml'
          namespace: $(NAMESPACE)

      # Install Helm on an agent machine
      - task: HelmInstaller@1
        inputs:
          helmVersionToInstall: 'latest'

      - task: HelmDeploy@0
        displayName: Helm package
        inputs:
          command: package
          save: false
          chartPath: 'deployment/entity-service'
          destination: $(Build.ArtifactStagingDirectory)
          updatedependency: true

      - task: KubernetesManifest@0
        name: bake
        displayName: Bake K8s manifests from Helm chart
        inputs:
          action: 'bake'
          helmChart: 'deployment/entity-service'
          namespace: $(NAMESPACE)
          releaseName: $(DEPLOYMENT)
          overrides: |
            global.postgresql.postgresqlPassword:notaproductionpassword
            api.ingress.enabled:false
            workers.replicaCount:4
            api.www.image.repository:$(frontendImageName)
            api.www.image.pullPolicy:Always
            api.app.image.repository:$(backendImageName)
            api.app.image.pullPolicy:Always
            api.dbinit.image.repository:$(backendImageName)
            workers.image.repository:$(backendImageName)
            workers.image.pullPolicy:Always
            anonlink.objectstore.uploadEnabled:true
            anonlink.objectstore.uploadSecure:false
            anonlink.objectstore.uploadAccessKey:testUploadAccessKey
            anonlink.objectstore.uploadSecretKey:testUploadSecretKey
            anonlink.objectstore.downloadSecure:false
            anonlink.objectstore.downloadAccessKey:testDownloadAccessKey
            anonlink.objectstore.downloadSecretKey:testDownloadSecretKey
            minio.accessKey:testMinioAccessKey
            minio.secretKey:testMinioSecretKey

      - task: KubernetesManifest@0
        displayName: Deploy K8s manifest
        inputs:
          action: 'deploy'
          failOnStderr: false
          kubernetesServiceConnection: '$(CLUSTER)'
          manifests: '$(bake.manifestsBundle)'
          namespace: $(NAMESPACE)
          containers: |
            data61/anonlink-nginx:$(DOCKER_TAG)
            data61/anonlink-app:$(DOCKER_TAG)
            #data61/anonlink-worker:$(DOCKER_TAG)

      # The file `.azurePipeline/k8s_test_job.yaml.tmpl` is a template for the creation of the test job.
      # The test job stops when all the tests have been run, creating a test result file in the PVC.
      - script: |
          cat .azurePipeline/k8s_test_job.yaml.tmpl | \
          sed 's|\$PVC'"|$(PVC)|g" | \
          sed 's|\$DEPLOYMENT_NAME'"|$(DEPLOYMENT)|g" | \
          sed 's|\$TEST_E2E_IMAGE_NAME_WITH_TAG'"|$(TEST_E2E_IMAGE_NAME_WITH_TAG)|g" | \
          sed 's|\$SERVICE'"|$(SERVICE)|g" > $(Build.ArtifactStagingDirectory)/k8s_test_job.yaml
        displayName: 'Prepare integration test job from template'

      - task: KubernetesManifest@0
        displayName: 'Deploy Integration Test Job'
        inputs:
          action: 'deploy'
          kubernetesServiceConnection: '$(CLUSTER)'
          manifests: '$(Build.ArtifactStagingDirectory)/k8s_test_job.yaml'
          namespace: $(NAMESPACE)

      - task: Kubernetes@1
        displayName: 'Wait for integration test pod'
        name: waitpods
        inputs:
          connectionType: 'Kubernetes Service Connection'
          kubernetesServiceEndpoint: $(CLUSTER)
          command: 'wait'
          arguments: '--namespace $(NAMESPACE) -ldeployment=$(DEPLOYMENT) --timeout=60s --for=condition=Ready pods'

      - task: Kubernetes@1
        displayName: 'Integration test logs'
        inputs:
          connectionType: 'Kubernetes Service Connection'
          kubernetesServiceEndpoint: $(CLUSTER)
          command: 'logs'
          arguments: '--namespace $(NAMESPACE) -ljobgroup=anonlink-integration-test -ldeployment=$(DEPLOYMENT) -f'

      - script: |
          echo "Get the test results"
          cat .azurePipeline/k8s_get_results.yaml.tmpl | \
          sed 's|\$PVC'"|$(PVC)|g" | \
          sed 's|\$DEPLOYMENT_NAME'"|$(DEPLOYMENT)|g" | \
          sed 's|\$POD_NAME'"|$(POD_NAME)|g" > $(Build.ArtifactStagingDirectory)/k8s_get_results.yaml
        displayName: 'Prepare get results pod from template'

      - task: KubernetesManifest@0
        displayName: 'Deploy Result fetching Pod'
        inputs:
          action: 'deploy'
          kubernetesServiceConnection: '$(CLUSTER)'
          manifests: '$(Build.ArtifactStagingDirectory)/k8s_get_results.yaml'
          namespace: $(NAMESPACE)

      - task: Kubernetes@1
        displayName: 'Copy integration test results'
        inputs:
          connectionType: 'Kubernetes Service Connection'
          kubernetesServiceEndpoint: $(CLUSTER)
          command: 'cp'
          arguments: '$(NAMESPACE)/$(POD_NAME):/mnt/results.xml k8s-results.xml'

      - script: |
          if [ -f "k8s-results.xml" ]; then
            echo "Result file 'k8s-results.xml' exists"
          else
            echo "Result file 'k8s-results.xml' does not exist"
            exit 1
          fi
        displayName: 'Check test result file existence'

      # Publish the test results.
      - task: PublishTestResults@2
        condition: succeeded()
        inputs:
          testResultsFormat: 'JUnit'
          testResultsFiles: 'k8s-results.xml'
          testRunTitle: 'Kubernetes Integration test results'
          failTaskOnFailedTests: true

      # Publish all the pods logs if some tests failed.
      - template: .azurePipeline/templatePublishLogsFromPods.yml
        parameters:
          logsFolderPath: $(Build.ArtifactStagingDirectory)
          releaseName: $(DEPLOYMENT)
          namespace: $(NAMESPACE)
          kubernetesServiceEndpoint: $(CLUSTER)
          JobAttempt: $(System.JobAttempt)

      # Cleanup
      - task: KubernetesManifest@0
        condition: always()
        inputs:
          action: 'delete'
          kubernetesServiceConnection: $(CLUSTER)
          arguments: '-f $(bake.manifestsBundle)'
          namespace: $(NAMESPACE)

      # Note that all the test resources (i.e. the pvc, the job and the pod have a label `deployment` set to $(DEPLOYMENT)
      - task: KubernetesManifest@0
        condition: always()
        inputs:
          action: 'delete'
          kubernetesServiceConnection: $(CLUSTER)
          arguments: 'pods,jobs,pvc,pv,statefulsets,deployment -l deployment=$(DEPLOYMENT)'
          namespace: $(NAMESPACE)