azure-pipelines.yml

variables:
  AZCOPY_GOLANG_VERSION: '1.23.1'
  AZCOPY_GOLANG_VERSION_COVERAGE: '1.23.1'

trigger:
  branches:
    include: [ 'main' ]

pr:
  branches:
    include: [ '*' ]
     
jobs:
  - job: Build
    timeoutInMinutes: 360
    strategy:
      matrix:
        Ubuntu-20:
          imageName: 'ubuntu-latest'
          type: 'linux'
        MacOS:
          imageName: 'macos-latest'
          type: 'mac-os'
        Windows:
          imageName: 'windows-latest'
          type: 'windows'
    pool:
      vmImage: $(imageName)
    steps:
      - task: GoTool@0
        env:
          GO111MODULE: 'on'
        inputs:
          version: $(AZCOPY_GOLANG_VERSION)

      - script: |
          echo 'Running GO Vet'
          go vet 
        displayName: 'Golang Vet - Linux'
        workingDirectory: $(System.DefaultWorkingDirectory)

      - script: |
          GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_amd64"
        displayName: 'Generate Linux AMD64'
        condition: eq(variables.type, 'linux')

      - script: |
          GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64"
        displayName: 'Generate Linux AMD64 SE Integration'
        condition: eq(variables.type, 'linux')

      - script: |
          GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64"
        displayName: 'Generate Linux ARM64'
        condition: eq(variables.type, 'linux')

      - script: |
          go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe"
        displayName: 'Generate Windows AMD64'
        condition: eq(variables.type, 'windows')
        env:
          GOARCH: amd64
          GOOS: windows
          CGO_ENABLED: 0

      - script: |
          go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe"
        displayName: 'Generate Windows i386'
        condition: eq(variables.type, 'windows')
        env:
          GOARCH: 386
          GOOS: windows
          CGO_ENABLED: 0

      - script: |
          go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe"
        displayName: 'Generate Windows ARM'
        condition: eq(variables.type, 'windows')
        env:
          GOARCH: arm
          GOARM: 7
          GOOS: windows
          CGO_ENABLED: 0

      - script: |
          cp NOTICE.txt $(Build.ArtifactStagingDirectory)
        displayName: 'Copy NOTICE.txt'
        condition: eq(variables.type, 'linux')

      - script: |
          CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64"
        displayName: 'Generate MacOS Build with AMD64'
        condition: eq(variables.type, 'mac-os')

      # uncomment below when manually releasing for m1
#      - script: |
#          curl -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64" -L "BUILD_URL"
#        displayName: 'Generate MacOS Build with ARM64'
#        condition: eq(variables.type, 'mac-os')

      # cross compile to make sure nothing broke
      - script: |
          GOARCH=arm64 CGO_ENABLED=1 go build
        displayName: 'Test Cross-compiled MacOS Build with ARM64'
        condition: eq(variables.type, 'mac-os')

      - task: PublishBuildArtifacts@1
        displayName: 'Publish Artifacts'
        condition: succeededOrFailed()
        
      - task: ComponentGovernanceComponentDetection@0
        inputs:
          scanType: 'Register'
          verbosity: 'Verbose'
          alertWarningLevel: 'High'

  - template: azurePipelineTemplates/run-e2e.yml
    parameters:
      name: 'Old_E2E'
      test_cli_param: '-tags olde2etest'

  - template: azurePipelineTemplates/run-e2e.yml
    parameters:
      name: 'New_E2E'
      test_cli_param: '-run "TestNewE2E/.*"'

  - job: Test_On_Ubuntu
    variables:
      isMutexSet: 'false'
    # allow maximum build time, in case we have build congestion
    timeoutInMinutes: 360
    pool:
      vmImage: 'ubuntu-latest'
    steps:
      - task: UsePythonVersion@0
        name: 'Set_up_Python'
        inputs:
          versionSpec: '3.7'
      - task: GoTool@0
        name: 'Set_up_Golang'
        inputs:
          version: $(AZCOPY_GOLANG_VERSION_COVERAGE)
      - task: DownloadSecureFile@1
        name: ciGCSServiceAccountKey
        displayName: 'Download GCS Service Account Key'
        inputs:
          secureFile: 'ci-gcs-dev.json'
      - script: |
          go install github.com/jstemmer/go-junit-report@v0.9.1
          go install github.com/axw/gocov/gocov@v1.1.0
          go install github.com/AlekSi/gocov-xml@v1.0.0
          go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b
        displayName: 'Install dependencies'
      - script: |
          pip install azure-storage-blob==12.12.0
          # set the variable to indicate that the mutex is being acquired
          # note: we set it before acquiring the mutex to ensure we release the mutex. 
          # setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the 
          # mutex but before we set this variable.
          # setting this before will always work since it is valid to break an un-acquired mutex.
          echo '##vso[task.setvariable variable=isMutexSet]true'
          # acquire the mutex before running live tests to avoid conflicts
          python ./tool_distributed_mutex.py lock "$(MUTEX_URL)"
        name: 'Acquire_the_distributed_mutex'
      - template: azurePipelineTemplates/run-ut.yml
        parameters:
          directory: 'cmd'
          coverage_name: 'cmd'
      - template: azurePipelineTemplates/run-ut.yml
        parameters:
          directory: 'common'
          coverage_name: 'common'
      - template: azurePipelineTemplates/run-ut.yml
        parameters:
          directory: 'common/parallel'
          coverage_name: 'parallel'
      - template: azurePipelineTemplates/run-ut.yml
        parameters:
          directory: 'ste'
          coverage_name: 'ste'
          with_federated_credential: true
      - template: azurePipelineTemplates/run-ut.yml
        parameters:
          directory: 'sddl'
          coverage_name: 'sddl'
      - script: |
          GOARCH=amd64 GOOS=linux go build -cover -tags "netgo" -o azcopy_linux_amd64 
          go build -tags "netgo" -o test-validator ./testSuite/
          mkdir test-temp
          mkdir coverage
          export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64
          export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator
          export TEST_DIRECTORY_PATH=$(pwd)/test-temp

          keyctl session test python ./testSuite/scripts/run.py
          
          echo 'Formatting coverage directory to legacy txt format'
          go tool covdata textfmt -i=coverage -o smoke_coverage.txt
          echo 'Formatting coverage to json format'
          $(go env GOPATH)/bin/gocov convert smoke_coverage.txt > smoke_coverage.json
          echo 'Formatting coverage to xml format'
          $(go env GOPATH)/bin/gocov-xml < smoke_coverage.json > smoke_coverage.xml
        name: 'Run_smoke_tests'
        env:
          ACCOUNT_NAME: $(ACCOUNT_NAME)
          ACCOUNT_KEY: $(ACCOUNT_KEY)
          AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
          AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
          GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT)
          GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath)
          ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID)
          AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
          CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL)
          CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL)
          CONTAINER_SAS_URL: $(CONTAINER_SAS_URL)
          FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL)
          FILESYSTEM_URL: $(FILESYSTEM_URL)
          OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT)
          OAUTH_TENANT_ID: $(OAUTH_TENANT_ID)
          PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL)
          S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL)
          S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL)
          S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL)
          S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL)
          S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL)
          SHARE_SAS_URL: $(SHARE_SAS_URL)
          GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage'
        condition: succeededOrFailed()
      # Smoke Tests Publishing
      - task: PublishCodeCoverageResults@1
        condition: succeededOrFailed()
        inputs:
          codeCoverageTool: Cobertura
          summaryFileLocation: $(System.DefaultWorkingDirectory)/**/smoke_coverage.xml
      - script: |
          pip install azure-storage-blob==12.12.0
          python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)"
        name: 'Release_the_distributed_mutex'
        # this runs even if the job was canceled (only if the mutex was acquired by this job)
        condition: and(always(), eq(variables['isMutexSet'], 'true'))

  - job: Additional_E2E_Test_Linux
    timeoutInMinutes: 360
    strategy:
      matrix:
        Linux:
          AgentName: "blobfuse-ubuntu20"
          build_name: 'azcopy_linux_amd64'
          container_name: 'testcontainer1'
          GOOS: 'linux'
          GOARCH: 'amd64'
          CGO_ENABLED: '0'
    pool:
      name: "blobfuse-ubuntu-pool"
      demands:
        - ImageOverride -equals $(AgentName)

    steps:
      - script: |
          # Install Az.Accounts module using Bash script
          sudo apt-get update
          sudo apt-get install -y powershell
          pwsh -Command "Install-Module -Name Az.Accounts -Scope CurrentUser -Repository PSGallery -AllowClobber -Force"
        displayName: 'Install PowerShell Az Module'

      - task: GoTool@0
        inputs:
          version: $(AZCOPY_GOLANG_VERSION_COVERAGE)
      
      # Install azcli
      - script: |
          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
          az --version
        displayName: 'Install Azure CLI'

      - template: azurePipelineTemplates/run_scenarios.yml
        parameters:
          container_name: $(container_name)
          storage_account_name: $(AZCOPY_E2E_ACCOUNT_NAME)
          orp_source_account_name: $(ORP_SOURCE_ACCOUNT_NAME)
          orp_destination_account_name: $(ORP_DESTINATION_ACCOUNT_NAME)
          orp_source_container_name: $(ORP_SOURCE_CONTAINER_NAME)
          destination_container_name: $(DESTINATION_CONTAINER_NAME)
          GOOS: $(GOOS)
          GOARCH: $(GOARCH)
          CGO_ENABLED: $(CGO_ENABLED)
          build_name: $(build_name)
          azcopy_msi_app_id: $(AZCOPY_MSI_APP_ID)
          
  - job: Additional_E2E_Test_Windows
    timeoutInMinutes: 360
    strategy:
      matrix:
        Windows:
          AgentName: 'azcopy-windows-22'
          build_name: 'azcopy_windows_amd64.exe'
          container_name: 'testcontainer2'
          GOOS: 'windows'
          GOARCH: 'amd64'
          CGO_ENABLED: '0'
    pool:
      name: "azcopy-windows-pool"
      demands:
        - ImageOverride -equals $(AgentName)

    steps:
      - task: PowerShell@2
        inputs:
          targetType: 'inline'
          script: |
            # Install Az.Accounts module using PowerShell
            Install-Module -Name Az.Accounts -Scope CurrentUser -Repository PSGallery -AllowClobber -Force
          displayName: 'Install PowerShell Az Module'

      - task: GoTool@0
        inputs:
          version: $(AZCOPY_GOLANG_VERSION_COVERAGE)

      - task: PowerShell@2
        displayName: 'Install Azure CLI'
        inputs:
          targetType: 'inline'
          script: |
            # Install Azure CLI using Chocolatey
            choco install azure-cli -y
            az --version
          
      - template: azurePipelineTemplates/run_scenarios.yml
        parameters:
          container_name: $(container_name)
          storage_account_name: $(AZCOPY_E2E_ACCOUNT_NAME)
          orp_source_account_name: $(ORP_SOURCE_ACCOUNT_NAME)
          orp_destination_account_name: $(ORP_DESTINATION_ACCOUNT_NAME)
          orp_source_container_name: $(ORP_SOURCE_CONTAINER_NAME)
          destination_container_name: $(DESTINATION_CONTAINER_NAME)
          GOOS: $(GOOS)
          GOARCH: $(GOARCH)
          CGO_ENABLED: $(CGO_ENABLED)
          build_name: $(build_name)
          azcopy_msi_app_id: $(AZCOPY_MSI_APP_ID)

    #TODO: Add MacOS E2E tests after creating macos pool