Add Testing of Conda Artifacts

doc/dev/conda-builds.md

@@ -15,9 +15,18 @@ A Conda Artifact defines:
 - Any other necessary details.
 
 ## How to Build an Azure SDK Conda Package Locally
+#### If using powershell, you will need to prep your environment before proceeding to the next step
 
+```
+powershell -ExecutionPolicy ByPass -NoExit -Command "& '<path-to-conda-folder>\shell\condabin\conda-hook.ps1' ; conda activate '<path-to-conda-folder>' "
+```
+
+Afterwards, invoke `conda init powershell` and re-create the pshell session.
+
+By default, your powershell environment will now load `conda`. If you want pure pip, you will need to use explicit invocations of your `python` locations to create virtual envs.
 ### Set up your conda environment
 
+
 You will notice that all the azure-sdk conda distributions have the **same** version number and requirement set. This is due to the fact that the azure-sdk team pushes our conda packages out in waves. To support this, all versions are set via a common environment variable `AZURESDK_CONDA_VERSION`.
 
 We keep this environment variable set properly across all our builds by using a common `conda_env.yml` when creating our build environment. This environment definition ensures that: 

eng/conda_env.yml

@@ -1,2 +1,2 @@
 variables:
-  AZURESDK_CONDA_VERSION: '2021.05.01'
+  AZURESDK_CONDA_VERSION: '2021.05.01b1'

eng/conda_test_requirements.txt

@@ -0,0 +1,8 @@
+aiohttp>=3.0; python_version >= '3.5'
+../../../tools/azure-devtools
+../../../tools/azure-sdk-tools
+mock;
+aiodns>=2.0; python_version >= '3.5'
+parameterized>=0.7.3; python_version >= '3.0'
+trio; python_version >= '3.5'
+typing_extensions>=3.7.2

eng/pipelines/templates/jobs/ci.tests.yml

@@ -31,6 +31,9 @@ parameters:
   - name: CloudConfig
     type: object
     default: {}
+  - name: CondaArtifacts
+    type: object
+    default: []
 
 jobs:
   - job:
@@ -110,3 +113,12 @@ jobs:
             parameters:
               ServiceDirectory: ${{ parameters.ServiceDirectory }}
               BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
+    - template: ../steps/test-conda.yml
+      parameters:
+        CondaArtifacts: ${{ parameters.CondaArtifacts }}
+        ServiceDirectory: ${{ parameters.ServiceDirectory }}
+        TestMarkArgument: ${{ parameters.TestMarkArgument }}
+        OSVmImage: $(OSVmImage)
+        PythonVersion: $(PythonVersion)
+

eng/pipelines/templates/jobs/ci.yml

@@ -123,6 +123,7 @@ jobs:
         TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
         ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
         InjectedPackages: ${{ parameters.InjectedPackages }}
+        CondaArtifacts: ${{ parameters.CondaArtifacts}}
 
   - job: 'RunRegression'
     condition: and(succeededOrFailed(), or(eq(variables['Run.Regression'], 'true'), and(eq(variables['Build.Reason'], 'Schedule'), eq(variables['System.TeamProject'],'internal'))))

eng/pipelines/templates/stages/archetype-conda-release.yml

@@ -9,7 +9,7 @@ parameters:
 stages:
   - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}:
     - ${{ each artifact in parameters.CondaArtifacts }}:
-      - stage: Release_${{ replace(artifact.name, '-', '_') }}
+      - stage: Release_${{ replace(artifact.name, '-', '_') }}_To_Blob
         displayName: 'Conda Release: ${{artifact.name}}'
         dependsOn: ${{parameters.DependsOn}}
         condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr'))

eng/pipelines/templates/steps/build-conda-artifacts.yml

@@ -74,9 +74,9 @@ steps:
 
     - bash: |
         source activate ${{ artifact.name }}
-        conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output" -c $(AzureSDKCondaChannel)
+        conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output/${{ artifact.name }}" -c $(AzureSDKCondaChannel)
       displayName: 'Activate Conda Environment and Build ${{ artifact.name }}'
-      workingDirectory: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
+      workingDirectory: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/conda-recipe
 
   - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
     parameters:

eng/pipelines/templates/steps/test-conda.yml

@@ -0,0 +1,146 @@
+parameters:
+  - name: TestPipeline
+    type: boolean
+    default: false
+  - name: ServiceDirectory
+    type: string
+    default: ''
+  - name: CondaArtifacts
+    type: object
+    default: []
+  - name: TestMarkArgument
+    type: string
+    default: ''
+  - name: PythonVersion
+    type: string
+    default: ''
+  - name: OSVmImage
+    type: string
+    default: ''
+
+steps:
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      artifactName: 'conda'
+      targetPath: $(Build.ArtifactStagingDirectory)
+
+  - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
+    parameters:
+      PackageName: "azure-template"
+      ServiceDirectory: "template"
+      TestPipeline: ${{ parameters.TestPipeline }}
+
+  - task: UsePythonVersion@0
+    displayName: 'Use Python $(PythonVersion)'
+    inputs:
+      versionSpec: $(PythonVersion)
+
+  - pwsh: |
+      # due to faulty deployed scripts/how the path gets manipulated by conda actions on 
+      # ubuntu and mac, we can't rely on bin/scripts being referenced correctly. see
+      # https://github.com/MicrosoftDocs/azure-devops-docs/issues/3812
+      $activateMethod = "source $($env:CONDA)/bin/activate"
+
+      # pypy3 is not a true python executable. in conda-land, we need to call it using pypy3, NOT python
+
+      # on windows, we need to add "--user" as otherwise pip won't successfully install/uninstall due to 
+      # how windows holds reservation on pip.exe. this is unnecessary on ubuntu/mac.
+      $requirementSuffix = ""
+
+      # we always want to prepend the path with conda bin   
+      Write-Host "##vso[task.prependpath]]$($env:CONDA)/bin"
+
+      if ($IsWindows) {
+        # powershell does not have an equivalent of call/source, which is necessary when
+        # using conda in azure devops. Note that we use `activate` natively here, as
+        # a later path prepend of the /scripts directory actually works.
+        $activateMethod = "call activate" 
+        $requirementSuffix = " --user"
+
+        # on windows only, need to prepend with the scripts directory as well
+        Write-Host "##vso[task.prependpath]$($env:CONDA)/Scripts"
+      }
+
+      if("$(PythonVersion)" -eq "pypy3"){
+        Write-Host "##vso[task.setvariable variable=PyVersion]-c conda-forge pypy3.7 pip"
+      }
+      else {
+        Write-Host "##vso[task.setvariable variable=PyVersion]python=$(PythonVersion)"
+      }
+
+      # we will use these variables extensively later
+      Write-Host "##vso[task.setvariable variable=activate.method]$activateMethod"
+      Write-Host "##vso[task.setvariable variable=requirement.suffix]$requirementSuffix"
+    displayName: 'Evaluate OS Specific PATH and Parameters'
+
+  - ${{ each artifact in parameters.CondaArtifacts }}:
+    # due to the fact that `pypy3` and `conda-build` conda packages are INCOMPATIBLE, we have to create
+    # a separate env to install `conda-build` and use that to `conda index` the local file channel
+    - script: |
+        echo "conda create --name ${{ artifact.name }} $(PyVersion) --yes"
+        conda create --name ${{ artifact.name }} $(PyVersion) --yes
+
+        echo "conda create --name index-env --yes"
+        conda create --name index-env --yes
+
+        echo "conda install --name index-env --yes --quiet conda-build"
+        conda install --name index-env --yes --quiet conda-build
+
+        echo "$(activate.method) index-env"
+        $(activate.method) index-env
+
+        echo "conda index $(Build.ArtifactStagingDirectory)/${{ artifact.name }}"
+        conda index $(Build.ArtifactStagingDirectory)/${{ artifact.name }}
+      displayName: 'Prepare Conda Environment for Testing ${{ artifact.name }}, Index the Target Local Artifact'
+
+    - script: |
+        echo "$(activate.method) ${{ artifact.name }}"
+        $(activate.method) ${{ artifact.name }}
+
+        echo "python -m pip install -r eng/ci_tools.txt $(requirement.suffix)"
+        python -m pip install -r eng/ci_tools.txt $(requirement.suffix)
+      displayName: 'Activate Conda Environment and Install General Dependencies ${{ artifact.name }}'
+
+    - pwsh: |
+        mkdir $(Agent.BuildDirectory)/conda/
+        Write-Host "##vso[task.setvariable variable=conda.build]$(Agent.BuildDirectory)/conda_checkout"
+      displayName: 'Create Conda Working Directory for Testing'
+
+    - ${{ each checkout in artifact.checkout }}:
+      - pwsh:
+          Write-Host "Clean up Conda Build Directory $(conda.build)"
+          Remove-Item $(conda.build)/* -Recurse -Force
+        displayName: 'Clean Up Before Testing ${{ artifact.name }}'
+
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+        parameters:
+          Paths:
+            - "${{ checkout.checkout_path }}/${{ checkout.package }}"
+            - "tools/"
+          Repositories:
+            - Name: "Azure/azure-sdk-for-python"
+              Commitish: "${{ checkout.Package }}_${{ checkout.Version }}"
+              WorkingDirectory: "$(conda.build)"
+          SkipDefaultCheckout: true
+
+      - script: |
+          echo "conda install --name ${{ artifact.name }} azure-identity -c $(AzureSDKChannel) --yes"
+          conda install --name ${{ artifact.name }} azure-identity -c $(AzureSDKChannel) --yes
+
+          echo "conda install --name ${{ artifact.name }} ${{ artifact.name }} -c $(Build.ArtifactStagingDirectory)/${{ artifact.name }} --yes"
+          conda install --name ${{ artifact.name }} ${{ artifact.name }} -c $(Build.ArtifactStagingDirectory)/${{ artifact.name }} --yes
+
+          echo "$(activate.method) ${{ artifact.name }}"
+          $(activate.method) ${{ artifact.name }}
+
+          echo "python -m pip install -r $(Build.SourcesDirectory)/eng/conda_test_requirements.txt"
+          python -m pip install -r $(Build.SourcesDirectory)/eng/conda_test_requirements.txt
+        displayName: 'Install ${{ checkout.package }} Conda Package and Dev Requirements'
+        workingDirectory: $(conda.build)/${{ checkout.checkout_path }}/${{ checkout.package }}
+
+      - script: |
+          echo "$(activate.method) ${{ artifact.name }}"
+          $(activate.method) ${{ artifact.name }}
+          python -m pytest .
+        displayName: 'Run Tests for ${{ checkout.package }}'
+        workingDirectory: $(conda.build)/${{ checkout.checkout_path }}/${{ checkout.package }}

scripts/devops_tasks/build_conda_artifacts.py

@@ -33,7 +33,7 @@
 
 VERSION_REGEX = re.compile(r"\s*AZURESDK_CONDA_VERSION\s*:\s*[\'](.*)[\']\s*")
 
-SUMMARY_TEMPLATE = "- Generated from {}."
+SUMMARY_TEMPLATE = " - Generated from {}."
 
 NAMESPACE_EXTENSION_TEMPLATE = """__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: str
 """

sdk/core/ci.yml

@@ -46,7 +46,7 @@ extends:
       safeName: azurecommon
     CondaArtifacts:
     - name: azure-core
-      meta_source: meta.yaml
+      meta_source: conda-recipe/meta.yaml
       common_root: azure
       checkout:
         - package: azure-core

sdk/storage/ci.yml

@@ -58,7 +58,7 @@ extends:
       safeName: azuremgmtstorageimportexport
     CondaArtifacts:
     - name: azure-storage
-      meta_source: meta.yaml
+      meta_source: conda-recipe/meta.yaml
       common_root: azure/storage
       checkout:
         - package: azure-storage-blob