Add Testing of Conda Artifacts

doc/dev/conda-builds.md

@@ -4,6 +4,8 @@
 
 Follow the instructions [here](https://docs.conda.io/projects/conda-build/en/latest/install-conda-build.html) to install `conda` and `conda-build`.
 
+**The Azure SDK Conda artifacts support `python3.8` and `python3.9` only.**
+
 ## CI Build Process
 
 There will be a `CondaArtifact` defined in the `ci.yml` of each service directory. (`sdk/<service>`)
@@ -15,9 +17,18 @@ A Conda Artifact defines:
 - Any other necessary details.
 
 ## How to Build an Azure SDK Conda Package Locally
+#### If using powershell, you will need to prep your environment before proceeding to the next step
+
+```
+powershell -ExecutionPolicy ByPass -NoExit -Command "& '<path-to-conda-folder>\shell\condabin\conda-hook.ps1' ; conda activate '<path-to-conda-folder>' "
+```
+
+Afterwards, invoke `conda init powershell` and re-create the pshell session.
 
+By default, your powershell environment will now load `conda`. If you want pure pip, you will need to use explicit invocations of your `python` locations to create virtual envs.
 ### Set up your conda environment
 
+
 You will notice that all the azure-sdk conda distributions have the **same** version number and requirement set. This is due to the fact that the azure-sdk team pushes our conda packages out in waves. To support this, all versions are set via a common environment variable `AZURESDK_CONDA_VERSION`.
 
 We keep this environment variable set properly across all our builds by using a common `conda_env.yml` when creating our build environment. This environment definition ensures that: 

eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml

@@ -31,9 +31,12 @@ parameters:
 - name: OsVmImage
   type: string
   default: MMSUbuntu18.04
+- name: GenerateJobName
+  type: string
+  default: 'generate_matrix'
 
 jobs:
-- job: generate_matrix
+- job: ${{ parameters.GenerateJobName }}
   variables:
     displayNameFilter: $[ coalesce(variables.jobMatrixFilter, '.*') ]
   pool:
@@ -89,8 +92,8 @@ jobs:
     - template: ${{ parameters.JobTemplatePath }}
       parameters:
         UsePlatformContainer: false
-        Matrix: dependencies.generate_matrix.outputs['generate_vm_job_matrix_${{ config.Name }}.matrix']
-        DependsOn: generate_matrix
+        Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['generate_vm_job_matrix_${{ config.Name }}.matrix']
+        DependsOn: ${{ parameters.GenerateJobName }}
         CloudConfig: ${{ parameters.CloudConfig }}
         ${{ each param in parameters.AdditionalParameters }}:
           ${{ param.key }}: ${{ param.value }}
@@ -99,8 +102,8 @@ jobs:
     - template: ${{ parameters.JobTemplatePath }}
       parameters:
         UsePlatformContainer: true
-        Matrix: dependencies.generate_matrix.outputs['generate_container_job_matrix_${{ config.Name }}.matrix']
-        DependsOn: generate_matrix
+        Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['generate_container_job_matrix_${{ config.Name }}.matrix']
+        DependsOn: ${{ parameters.GenerateJobName }}
         CloudConfig: ${{ parameters.CloudConfig }}
         ${{ each param in parameters.AdditionalParameters }}:
           ${{ param.key }}: ${{ param.value }}

eng/conda_env.yml

@@ -1,2 +1,2 @@
 variables:
-  AZURESDK_CONDA_VERSION: '2021.05.01'
+  AZURESDK_CONDA_VERSION: '2021.05.01b1'

eng/conda_test_requirements.txt

@@ -0,0 +1,11 @@
+aiohttp>=3.0; python_version >= '3.5'
+../../../tools/azure-devtools
+../../../tools/azure-sdk-tools
+mock;
+aiodns>=2.0; python_version >= '3.5'
+parameterized>=0.7.3; python_version >= '3.0'
+trio; python_version >= '3.5'
+typing_extensions>=3.7.2
+futures==3.3.0; python_version <= '2.7'
+cryptography
+adal

eng/pipelines/templates/jobs/ci.conda.tests.yml

@@ -0,0 +1,187 @@
+parameters:
+  - name: TestPipeline
+    type: boolean
+    default: false
+  - name: ServiceDirectory
+    type: string
+    default: ''
+  - name: CondaArtifacts
+    type: object
+    default: []
+  - name: TestMarkArgument
+    type: string
+    default: ''
+  - name: PythonVersion
+    type: string
+    default: ''
+  - name: OSVmImage
+    type: string
+    default: ''
+  - name: Matrix
+    type: string
+  - name: DependsOn
+    type: string
+    default: ''
+  - name: UsePlatformContainer
+    type: boolean
+    default: false
+  - name: TestTimeoutInMinutes
+    type: number
+    default: 0
+  - name: CloudConfig
+    type: object
+    default: {}
+
+jobs:
+  - job:
+    displayName: 'Test Conda'
+    condition: |
+      and(
+        succeededOrFailed(),
+        ne(variables['Skip.TestConda'], 'true')
+      )
+    timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
+
+    dependsOn:
+      - ${{ parameters.DependsOn }}
+
+    strategy:
+      matrix: $[ ${{ parameters.Matrix }} ]
+
+    pool:
+      name: $(Pool)
+      vmImage: $(OSVmImage)
+
+    ${{ if eq(parameters.UsePlatformContainer, 'true') }}:
+      # Add a default so the job doesn't fail when the matrix is empty
+      container: $[ variables['Container'] ]
+
+    variables:
+    - template: ../variables/globals.yml
+
+    steps:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifactName: 'conda'
+        targetPath: $(Build.ArtifactStagingDirectory)
+
+    - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
+      parameters:
+        PackageName: "azure-template"
+        ServiceDirectory: "template"
+        TestPipeline: ${{ parameters.TestPipeline }}
+
+    - task: UsePythonVersion@0
+      displayName: 'Use Python $(PythonVersion)'
+      inputs:
+        versionSpec: $(PythonVersion)
+
+    - pwsh: |
+        # due to faulty deployed scripts/how the path gets manipulated by conda actions on 
+        # ubuntu and mac, we can't rely on bin/scripts being referenced correctly. see
+        # https://github.com/MicrosoftDocs/azure-devops-docs/issues/3812
+        $activateMethod = "source $($env:CONDA)/bin/activate"
+
+        # pypy3 is not a true python executable. in conda-land, we need to call it using pypy3, NOT python
+
+        # on windows, we need to add "--user" as otherwise pip won't successfully install/uninstall due to 
+        # how windows holds reservation on pip.exe. this is unnecessary on ubuntu/mac.
+        $requirementSuffix = ""
+
+        # we always want to prepend the path with conda bin   
+        Write-Host "##vso[task.prependpath]]$($env:CONDA)/bin"
+
+        if ($IsWindows) {
+          # powershell does not have an equivalent of call/source, which is necessary when
+          # using conda in azure devops. Note that we use `activate` natively here, as
+          # a later path prepend of the /scripts directory actually works.
+          $activateMethod = "call activate" 
+          $requirementSuffix = " --user"
+
+          # on windows only, need to prepend with the scripts directory as well
+          Write-Host "##vso[task.prependpath]$($env:CONDA)/Scripts"
+        }
+
+        if("$(PythonVersion)" -eq "pypy3"){
+          Write-Host "##vso[task.setvariable variable=PyVersion]-c conda-forge pypy3.7 pip"
+        }
+        else {
+          Write-Host "##vso[task.setvariable variable=PyVersion]python=$(PythonVersion)"
+        }
+
+        # we will use these variables extensively later
+        Write-Host "##vso[task.setvariable variable=activate.method]$activateMethod"
+        Write-Host "##vso[task.setvariable variable=requirement.suffix]$requirementSuffix"
+      displayName: 'Evaluate OS Specific PATH and Parameters'
+
+    - ${{ each artifact in parameters.CondaArtifacts }}:
+      # due to the fact that `pypy3` and `conda-build` conda packages are INCOMPATIBLE, we have to create
+      # a separate env to install `conda-build` and use that to `conda index` the local file channel
+      - script: |
+          echo "conda create --name ${{ artifact.name }} $(PyVersion) --yes"
+          conda create --name ${{ artifact.name }} $(PyVersion) --yes
+
+          echo "conda create --name index-env --yes"
+          conda create --name index-env --yes
+
+          echo "conda install --name index-env --yes --quiet conda-build"
+          conda install --name index-env --yes --quiet conda-build
+
+          echo "$(activate.method) index-env"
+          $(activate.method) index-env
+
+          echo "conda index $(Build.ArtifactStagingDirectory)/${{ artifact.name }}"
+          conda index $(Build.ArtifactStagingDirectory)/${{ artifact.name }}
+        displayName: 'Prepare Conda Environment for Testing ${{ artifact.name }}, Index the Target Local Artifact'
+
+      - script: |
+          echo "$(activate.method) ${{ artifact.name }}"
+          $(activate.method) ${{ artifact.name }}
+
+          echo "python -m pip install -r eng/ci_tools.txt $(requirement.suffix)"
+          python -m pip install -r eng/ci_tools.txt $(requirement.suffix)
+        displayName: 'Activate Conda Environment and Install General Dependencies ${{ artifact.name }}'
+
+      - pwsh: |
+          mkdir $(Agent.BuildDirectory)/conda/
+          Write-Host "##vso[task.setvariable variable=conda.build]$(Agent.BuildDirectory)/conda_checkout"
+        displayName: 'Create Conda Working Directory for Testing'
+
+      - ${{ each checkout in artifact.checkout }}:
+        - pwsh:
+            Write-Host "Clean up Conda Build Directory $(conda.build)"
+            Remove-Item $(conda.build)/* -Recurse -Force
+          displayName: 'Clean Up Before Testing ${{ artifact.name }}'
+
+        - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+          parameters:
+            Paths:
+              - "${{ checkout.checkout_path }}"
+              - "sdk/conftest.py"
+              - "tools/"
+            Repositories:
+              - Name: "Azure/azure-sdk-for-python"
+                Commitish: "${{ checkout.Package }}_${{ checkout.Version }}"
+                WorkingDirectory: "$(conda.build)"
+            SkipDefaultCheckout: true
+
+        - script: |
+            echo "conda install --name ${{ artifact.name }} azure-identity -c $(AzureSDKCondaChannel) --yes"
+            conda install --name ${{ artifact.name }} azure-identity -c $(AzureSDKCondaChannel) --yes
+
+            echo "conda install --name ${{ artifact.name }} ${{ artifact.name }} -c $(Build.ArtifactStagingDirectory)/${{ artifact.name }} --yes -c $(AzureSDKCondaChannel)"
+            conda install --name ${{ artifact.name }} ${{ artifact.name }} -c $(Build.ArtifactStagingDirectory)/${{ artifact.name }} --yes -c $(AzureSDKCondaChannel)
+
+            echo "$(activate.method) ${{ artifact.name }}"
+            $(activate.method) ${{ artifact.name }}
+
+            echo "python -m pip install -r $(Build.SourcesDirectory)/eng/conda_test_requirements.txt"
+            python -m pip install -r $(Build.SourcesDirectory)/eng/conda_test_requirements.txt
+          displayName: 'Install ${{ checkout.package }} Conda Package and Dev Requirements'
+          workingDirectory: $(conda.build)/${{ checkout.checkout_path }}/${{ checkout.package }}
+
+        - script: |
+            echo "$(activate.method) ${{ artifact.name }}"
+            $(activate.method) ${{ artifact.name }}
+            python -m pytest $(conda.build)/${{ checkout.checkout_path }}/${{ checkout.package }}
+          displayName: 'Run Tests for ${{ checkout.package }}'

eng/pipelines/templates/jobs/ci.yml

@@ -126,6 +126,29 @@ jobs:
         ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
         InjectedPackages: ${{ parameters.InjectedPackages }}
 
+  - ${{ if gt(length(parameters.CondaArtifacts), 0) }}:
+    - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
+      parameters:
+        JobTemplatePath: /eng/pipelines/templates/jobs/ci.conda.tests.yml
+        GenerateJobName: generate_conda_matrix
+        DependsOn:
+          - 'Build'
+        MatrixConfigs:
+            - Name: Python_ci_conda_envs
+              Path: eng/pipelines/templates/stages/platform-matrix-conda-support.json
+              Selection: sparse
+              GenerateVMJobs: true
+        MatrixFilters: ${{ parameters.MatrixFilters }}
+        MatrixReplace: ${{ parameters.MatrixReplace }}
+        CloudConfig:
+          Cloud: Public
+        AdditionalParameters:
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          TestPipeline: ${{ parameters.TestPipeline }}
+          TestMarkArgument: ${{ parameters.TestMarkArgument }}
+          TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
+          CondaArtifacts: ${{ parameters.CondaArtifacts}}
+
   - job: 'RunRegression'
     condition: and(succeededOrFailed(), or(eq(variables['Run.Regression'], 'true'), and(eq(variables['Build.Reason'], 'Schedule'), eq(variables['System.TeamProject'],'internal'))))
     displayName: 'Run Regression'

eng/pipelines/templates/stages/archetype-conda-release.yml

@@ -9,7 +9,7 @@ parameters:
 stages:
   - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}:
     - ${{ each artifact in parameters.CondaArtifacts }}:
-      - stage: Release_${{ replace(artifact.name, '-', '_') }}
+      - stage: Release_${{ replace(artifact.name, '-', '_') }}_To_Blob
         displayName: 'Conda Release: ${{artifact.name}}'
         dependsOn: ${{parameters.DependsOn}}
         condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr'))

eng/pipelines/templates/stages/platform-matrix-conda-support.json

@@ -0,0 +1,10 @@
+{
+  "matrix": {
+    "Agent": {
+      "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" },
+      "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" },
+      "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" }
+    },
+    "PythonVersion": [ "3.6", "3.8", "3.9" ]
+  }
+}

eng/pipelines/templates/steps/build-conda-artifacts.yml

@@ -74,9 +74,9 @@ steps:
 
     - bash: |
         source activate ${{ artifact.name }}
-        conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output" -c $(AzureSDKCondaChannel)
+        conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output/${{ artifact.name }}" -c $(AzureSDKCondaChannel)
       displayName: 'Activate Conda Environment and Build ${{ artifact.name }}'
-      workingDirectory: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
+      workingDirectory: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/conda-recipe
 
   - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
     parameters:

scripts/devops_tasks/build_conda_artifacts.py

@@ -33,7 +33,7 @@
 
 VERSION_REGEX = re.compile(r"\s*AZURESDK_CONDA_VERSION\s*:\s*[\'](.*)[\']\s*")
 
-SUMMARY_TEMPLATE = "- Generated from {}."
+SUMMARY_TEMPLATE = " - Generated from {}."
 
 NAMESPACE_EXTENSION_TEMPLATE = """__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: str
 """

sdk/core/ci.yml

@@ -46,7 +46,7 @@ extends:
       safeName: azurecommon
     CondaArtifacts:
     - name: azure-core
-      meta_source: meta.yaml
+      meta_source: conda-recipe/meta.yaml
       common_root: azure
       checkout:
         - package: azure-core

sdk/storage/ci.yml

@@ -58,7 +58,7 @@ extends:
       safeName: azuremgmtstorageimportexport
     CondaArtifacts:
     - name: azure-storage
-      meta_source: meta.yaml
+      meta_source: conda-recipe/meta.yaml
       common_root: azure/storage
       checkout:
         - package: azure-storage-blob