From d4b3f3ecd344d2ca5d5571aa7eda7307a31a3c8c Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 26 Oct 2021 11:10:35 -0700 Subject: [PATCH 1/4] Unparallelize tests --- testing/Test.ps1 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/testing/Test.ps1 b/testing/Test.ps1 index c4e041d1974..db06668a6b7 100644 --- a/testing/Test.ps1 +++ b/testing/Test.ps1 @@ -2,6 +2,7 @@ param ( [string] $Path, [switch] $SkipInstall, [switch] $CI, + [switch] $ParallelCI, [switch] $OnlyPublicTests, [Parameter(Mandatory=$True)] @@ -74,7 +75,7 @@ if ($Type -eq 'k8s-extension') { $testFilePaths = "$PSScriptRoot/test/configurations" } -if ($CI) { +if ($ParallelCI) { # This runs the tests in parallel during the CI pipline to speed up testing Write-Host "Invoking Pester to run tests from '$testFilePath'..." @@ -113,6 +114,10 @@ if ($CI) { $failedJobs throw "One or more tests failed" } +} elseif ($CI) { + Write-Host "Invoking Pester to run tests from '$testFilePath'..." + $testResult = Invoke-Pester $testFilePath -Passthru -Output Detailed + $testResult | Export-JUnitReport -Path TestResults.xml } else { if ($Path) { Write-Host "Invoking Pester to run tests from '$PSScriptRoot/$Path'" From 6543416cb168b3cf11a30a5fd4c79726a27bda61 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 26 Oct 2021 11:12:39 -0700 Subject: [PATCH 2/4] Moved location of pipeline file --- testing/Test.ps1 | 4 +- .../pipeline/k8s-custom-pipelines.yml | 686 +++++++++--------- 2 files changed, 345 insertions(+), 345 deletions(-) rename k8s-custom-pipelines.yml => testing/pipeline/k8s-custom-pipelines.yml (97%) diff --git a/testing/Test.ps1 b/testing/Test.ps1 index db06668a6b7..a37287c4c33 100644 --- a/testing/Test.ps1 +++ b/testing/Test.ps1 @@ -41,9 +41,9 @@ if ($Type -eq 'k8s-extension') { } } if ($OnlyPublicTests) { - $testFilePaths = "$PSScriptRoot/test/extensions/public" + $testFilePath = "$PSScriptRoot/test/extensions/public" } else { - $testFilePaths = "$PSScriptRoot/test/extensions/public", "$PSScriptRoot/test/extensions/private-preview" + $testFilePath = "$PSScriptRoot/test/extensions" } } elseif ($Type -eq 'k8s-extension-private') { $k8sExtensionPrivateVersion = $ENVCONFIG.extensionVersion.'k8s-extension-private' diff --git a/k8s-custom-pipelines.yml b/testing/pipeline/k8s-custom-pipelines.yml similarity index 97% rename from k8s-custom-pipelines.yml rename to testing/pipeline/k8s-custom-pipelines.yml index 9ae6e96408f..7fb8b43a086 100644 --- a/k8s-custom-pipelines.yml +++ b/testing/pipeline/k8s-custom-pipelines.yml @@ -1,344 +1,344 @@ -trigger: - batch: true - branches: - include: - - k8s-extension/public - - k8s-extension/private -pr: - branches: - include: - - k8s-extension/public - - k8s-extension/private - -stages: -- stage: BuildTestPublishExtension - displayName: "Build, Test, and Publish Extension" - variables: - TEST_PATH: $(Agent.BuildDirectory)/s/testing - CLI_REPO_PATH: $(Agent.BuildDirectory)/s - SUBSCRIPTION_ID: "15c06b1b-01d6-407b-bb21-740b8617dea3" - RESOURCE_GROUP: "K8sPartnerExtensionTest" - BASE_CLUSTER_NAME: "k8s-extension-cluster" - IS_PRIVATE_BRANCH: $[or(eq(variables['Build.SourceBranch'], 'refs/heads/k8s-extension/private'), eq(variables['System.PullRequest.TargetBranch'], 'k8s-extension/private'))] - - EXTENSION_NAME: "k8s-extension" - EXTENSION_FILE_NAME: "k8s_extension" - jobs: - - job: K8sExtensionTestSuite - displayName: "Run the Test Suite" - pool: - vmImage: 'ubuntu-latest' - steps: - - checkout: self - - bash: | - echo "Installing helm3" - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 get_helm.sh - ./get_helm.sh --version v3.6.3 - - echo "Installing kubectl" - curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" - chmod +x ./kubectl - sudo mv ./kubectl /usr/local/bin/kubectl - kubectl version --client - displayName: "Setup the VM with helm3 and kubectl" - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: | - set -ev - echo "Building extension ${EXTENSION_NAME}..." - - # prepare and activate virtualenv - pip install virtualenv - python3 -m venv env/ - source env/bin/activate - - # clone azure-cli - git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli - - pip install --upgrade pip - pip install -q azdev - - ls $(CLI_REPO_PATH) - - azdev --version - azdev setup -c ../azure-cli -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME) - azdev extension build $(EXTENSION_NAME) - workingDirectory: $(CLI_REPO_PATH) - displayName: "Setup and Build Extension with azdev" - - - bash: | - K8S_EXTENSION_VERSION=$(ls ${EXTENSION_FILE_NAME}* | cut -d "-" -f2) - echo "##vso[task.setvariable variable=K8S_EXTENSION_VERSION]$K8S_EXTENSION_VERSION" - cp * $(TEST_PATH)/bin - workingDirectory: $(CLI_REPO_PATH)/dist - displayName: "Copy the Built .whl to Extension Test Path" - - - bash: | - RAND_STR=$RANDOM - AKS_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-aks" - ARC_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-arc" - - JSON_STRING=$(jq -n \ - --arg SUB_ID "$SUBSCRIPTION_ID" \ - --arg RG "$RESOURCE_GROUP" \ - --arg AKS_CLUSTER_NAME "$AKS_CLUSTER_NAME" \ - --arg ARC_CLUSTER_NAME "$ARC_CLUSTER_NAME" \ - --arg K8S_EXTENSION_VERSION "$K8S_EXTENSION_VERSION" \ - '{subscriptionId: $SUB_ID, resourceGroup: $RG, aksClusterName: $AKS_CLUSTER_NAME, arcClusterName: $ARC_CLUSTER_NAME, extensionVersion: {"k8s-extension": $K8S_EXTENSION_VERSION, connectedk8s: "1.0.0"}}') - echo $JSON_STRING > settings.json - cat settings.json - workingDirectory: $(TEST_PATH) - displayName: "Generate a settings.json file" - - - bash : | - echo "Downloading the kind script" - curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 - chmod +x ./kind - ./kind create cluster - displayName: "Create and Start the Kind cluster" - - - bash: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - displayName: "Upgrade az to latest version" - - - task: AzureCLI@2 - displayName: Bootstrap - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Bootstrap.ps1 -CI - workingDirectory: $(TEST_PATH) - - - task: AzureCLI@2 - displayName: Run the Test Suite Public Extensions Only - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Test.ps1 -CI -OnlyPublicTests -Type k8s-extension - workingDirectory: $(TEST_PATH) - continueOnError: true - condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'False')) - - - task: AzureCLI@2 - displayName: Run the Test Suite on Private + Public Extensions - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Test.ps1 -CI -Type k8s-extension - workingDirectory: $(TEST_PATH) - continueOnError: true - condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'True')) - - - task: PublishTestResults@2 - inputs: - testResultsFormat: 'JUnit' - testResultsFiles: '**/testing/results/*.xml' - failTaskOnFailedTests: true - condition: succeededOrFailed() - - - task: AzureCLI@2 - displayName: Cleanup - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Cleanup.ps1 -CI - workingDirectory: $(TEST_PATH) - condition: succeededOrFailed() - - - job: BuildPublishExtension - pool: - vmImage: 'ubuntu-latest' - displayName: "Build and Publish the Extension Artifact" - variables: - CLI_REPO_PATH: $(Agent.BuildDirectory)/s - steps: - - bash: | - echo "Using the private preview of k8s-extension to build..." - - cp $(CLI_REPO_PATH)/src/k8s-extension $(CLI_REPO_PATH)/src/k8s-extension-private -r - mv $(CLI_REPO_PATH)/src/k8s-extension-private/azext_k8s_extension $(CLI_REPO_PATH)/src/k8s-extension-private/azext_k8s_extension_private - cp $(CLI_REPO_PATH)/src/k8s-extension-private/setup_private.py $(CLI_REPO_PATH)/src/k8s-extension-private/setup.py - cp $(CLI_REPO_PATH)/src/k8s-extension-private/azext_k8s_extension_private/consts_private.py $(CLI_REPO_PATH)/src/k8s-extension-private/azext_k8s_extension_private/consts.py - - EXTENSION_NAME="k8s-extension-private" - EXTENSION_FILE_NAME="k8s_extension_private" - - echo "##vso[task.setvariable variable=EXTENSION_NAME]$EXTENSION_NAME" - echo "##vso[task.setvariable variable=EXTENSION_FILE_NAME]$EXTENSION_FILE_NAME" - condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'True')) - displayName: "Copy Files, Set Variables for k8s-extension-private" - - bash: | - echo "Using the public version of k8s-extension to build..." - - EXTENSION_NAME="k8s-extension" - EXTENSION_FILE_NAME="k8s_extension" - - echo "##vso[task.setvariable variable=EXTENSION_NAME]$EXTENSION_NAME" - echo "##vso[task.setvariable variable=EXTENSION_FILE_NAME]$EXTENSION_FILE_NAME" - condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'False')) - displayName: "Copy Files, Set Variables for k8s-extension" - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: | - set -ev - echo "Building extension ${EXTENSION_NAME}..." - - # prepare and activate virtualenv - pip install virtualenv - python3 -m venv env/ - source env/bin/activate - - # clone azure-cli - git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli - - pip install --upgrade pip - pip install -q azdev - - ls $(CLI_REPO_PATH) - - azdev --version - azdev setup -c ../azure-cli -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME) - azdev extension build $(EXTENSION_NAME) - workingDirectory: $(CLI_REPO_PATH) - displayName: "Setup and Build Extension with azdev" - - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: $(CLI_REPO_PATH)/dist - -- stage: AzureCLIOfficial - displayName: "Azure Official CLI Code Checks" - dependsOn: [] - jobs: - - job: CheckLicenseHeader - displayName: "Check License" - pool: - vmImage: 'ubuntu-latest' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: | - set -ev - - # prepare and activate virtualenv - python -m venv env/ - - chmod +x ./env/bin/activate - source ./env/bin/activate - - # clone azure-cli - git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli - - pip install --upgrade pip - pip install -q azdev - - azdev setup -c ../azure-cli -r ./ - - azdev --version - az --version - - azdev verify license - - - job: StaticAnalysis - displayName: "Static Analysis" - pool: - vmImage: 'ubuntu-latest' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: pip install wheel==0.30.0 pylint==1.9.5 flake8==3.5.0 requests - displayName: 'Install wheel, pylint, flake8, requests' - - bash: python scripts/ci/source_code_static_analysis.py - displayName: "Static Analysis" - - - job: IndexVerify - displayName: "Verify Extensions Index" - pool: - vmImage: 'ubuntu-latest' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - bash: | - #!/usr/bin/env bash - set -ev - pip install wheel==0.30.0 requests packaging - export CI="ADO" - python ./scripts/ci/test_index.py -v - displayName: "Verify Extensions Index" - - - job: SourceTests - displayName: "Integration Tests, Build Tests" - pool: - vmImage: 'ubuntu-latest' - strategy: - matrix: - Python36: - python.version: '3.6' - Python38: - python.version: '3.8' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: pip install wheel==0.30.0 - displayName: 'Install wheel==0.30.0' - - bash: ./scripts/ci/test_source.sh - displayName: 'Run integration test and build test' - env: - ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId) - ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - - - job: LintModifiedExtensions - displayName: "CLI Linter on Modified Extensions" - pool: - vmImage: 'ubuntu-latest' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: | - set -ev - - # prepare and activate virtualenv - pip install virtualenv - python -m virtualenv venv/ - source ./venv/bin/activate - - # clone azure-cli - git clone --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli - - pip install --upgrade pip - pip install azdev - - azdev --version - - azdev setup -c ../azure-cli -r ./ -e k8s-extension - - # overwrite the default AZURE_EXTENSION_DIR set by ADO - AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version - - AZURE_EXTENSION_DIR=~/.azure/cliextensions azdev linter --include-whl-extensions k8s-extension - displayName: "CLI Linter on Modified Extension" - env: - ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId) +trigger: + batch: true + branches: + include: + - k8s-extension/public + - k8s-extension/private +pr: + branches: + include: + - k8s-extension/public + - k8s-extension/private + +stages: +- stage: BuildTestPublishExtension + displayName: "Build, Test, and Publish Extension" + variables: + TEST_PATH: $(Agent.BuildDirectory)/s/testing + CLI_REPO_PATH: $(Agent.BuildDirectory)/s + SUBSCRIPTION_ID: "15c06b1b-01d6-407b-bb21-740b8617dea3" + RESOURCE_GROUP: "K8sPartnerExtensionTest" + BASE_CLUSTER_NAME: "k8s-extension-cluster" + IS_PRIVATE_BRANCH: $[or(eq(variables['Build.SourceBranch'], 'refs/heads/k8s-extension/private'), eq(variables['System.PullRequest.TargetBranch'], 'k8s-extension/private'))] + + EXTENSION_NAME: "k8s-extension" + EXTENSION_FILE_NAME: "k8s_extension" + jobs: + - job: K8sExtensionTestSuite + displayName: "Run the Test Suite" + pool: + vmImage: 'ubuntu-latest' + steps: + - checkout: self + - bash: | + echo "Installing helm3" + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh --version v3.6.3 + + echo "Installing kubectl" + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + kubectl version --client + displayName: "Setup the VM with helm3 and kubectl" + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: | + set -ev + echo "Building extension ${EXTENSION_NAME}..." + + # prepare and activate virtualenv + pip install virtualenv + python3 -m venv env/ + source env/bin/activate + + # clone azure-cli + git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install --upgrade pip + pip install -q azdev + + ls $(CLI_REPO_PATH) + + azdev --version + azdev setup -c ../azure-cli -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME) + azdev extension build $(EXTENSION_NAME) + workingDirectory: $(CLI_REPO_PATH) + displayName: "Setup and Build Extension with azdev" + + - bash: | + K8S_EXTENSION_VERSION=$(ls ${EXTENSION_FILE_NAME}* | cut -d "-" -f2) + echo "##vso[task.setvariable variable=K8S_EXTENSION_VERSION]$K8S_EXTENSION_VERSION" + cp * $(TEST_PATH)/bin + workingDirectory: $(CLI_REPO_PATH)/dist + displayName: "Copy the Built .whl to Extension Test Path" + + - bash: | + RAND_STR=$RANDOM + AKS_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-aks" + ARC_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-arc" + + JSON_STRING=$(jq -n \ + --arg SUB_ID "$SUBSCRIPTION_ID" \ + --arg RG "$RESOURCE_GROUP" \ + --arg AKS_CLUSTER_NAME "$AKS_CLUSTER_NAME" \ + --arg ARC_CLUSTER_NAME "$ARC_CLUSTER_NAME" \ + --arg K8S_EXTENSION_VERSION "$K8S_EXTENSION_VERSION" \ + '{subscriptionId: $SUB_ID, resourceGroup: $RG, aksClusterName: $AKS_CLUSTER_NAME, arcClusterName: $ARC_CLUSTER_NAME, extensionVersion: {"k8s-extension": $K8S_EXTENSION_VERSION, connectedk8s: "1.0.0"}}') + echo $JSON_STRING > settings.json + cat settings.json + workingDirectory: $(TEST_PATH) + displayName: "Generate a settings.json file" + + - bash : | + echo "Downloading the kind script" + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 + chmod +x ./kind + ./kind create cluster + displayName: "Create and Start the Kind cluster" + + - bash: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + displayName: "Upgrade az to latest version" + + - task: AzureCLI@2 + displayName: Bootstrap + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Bootstrap.ps1 -CI + workingDirectory: $(TEST_PATH) + + - task: AzureCLI@2 + displayName: Run the Test Suite Public Extensions Only + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Test.ps1 -CI -OnlyPublicTests -Type k8s-extension + workingDirectory: $(TEST_PATH) + continueOnError: true + condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'False')) + + - task: AzureCLI@2 + displayName: Run the Test Suite on Private + Public Extensions + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Test.ps1 -CI -Type k8s-extension + workingDirectory: $(TEST_PATH) + continueOnError: true + condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'True')) + + - task: PublishTestResults@2 + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '**/testing/results/*.xml' + failTaskOnFailedTests: true + condition: succeededOrFailed() + + - task: AzureCLI@2 + displayName: Cleanup + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Cleanup.ps1 -CI + workingDirectory: $(TEST_PATH) + condition: succeededOrFailed() + + - job: BuildPublishExtension + pool: + vmImage: 'ubuntu-latest' + displayName: "Build and Publish the Extension Artifact" + variables: + CLI_REPO_PATH: $(Agent.BuildDirectory)/s + steps: + - bash: | + echo "Using the private preview of k8s-extension to build..." + + cp $(CLI_REPO_PATH)/src/k8s-extension $(CLI_REPO_PATH)/src/k8s-extension-private -r + mv $(CLI_REPO_PATH)/src/k8s-extension-private/azext_k8s_extension $(CLI_REPO_PATH)/src/k8s-extension-private/azext_k8s_extension_private + cp $(CLI_REPO_PATH)/src/k8s-extension-private/setup_private.py $(CLI_REPO_PATH)/src/k8s-extension-private/setup.py + cp $(CLI_REPO_PATH)/src/k8s-extension-private/azext_k8s_extension_private/consts_private.py $(CLI_REPO_PATH)/src/k8s-extension-private/azext_k8s_extension_private/consts.py + + EXTENSION_NAME="k8s-extension-private" + EXTENSION_FILE_NAME="k8s_extension_private" + + echo "##vso[task.setvariable variable=EXTENSION_NAME]$EXTENSION_NAME" + echo "##vso[task.setvariable variable=EXTENSION_FILE_NAME]$EXTENSION_FILE_NAME" + condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'True')) + displayName: "Copy Files, Set Variables for k8s-extension-private" + - bash: | + echo "Using the public version of k8s-extension to build..." + + EXTENSION_NAME="k8s-extension" + EXTENSION_FILE_NAME="k8s_extension" + + echo "##vso[task.setvariable variable=EXTENSION_NAME]$EXTENSION_NAME" + echo "##vso[task.setvariable variable=EXTENSION_FILE_NAME]$EXTENSION_FILE_NAME" + condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'False')) + displayName: "Copy Files, Set Variables for k8s-extension" + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: | + set -ev + echo "Building extension ${EXTENSION_NAME}..." + + # prepare and activate virtualenv + pip install virtualenv + python3 -m venv env/ + source env/bin/activate + + # clone azure-cli + git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install --upgrade pip + pip install -q azdev + + ls $(CLI_REPO_PATH) + + azdev --version + azdev setup -c ../azure-cli -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME) + azdev extension build $(EXTENSION_NAME) + workingDirectory: $(CLI_REPO_PATH) + displayName: "Setup and Build Extension with azdev" + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: $(CLI_REPO_PATH)/dist + +- stage: AzureCLIOfficial + displayName: "Azure Official CLI Code Checks" + dependsOn: [] + jobs: + - job: CheckLicenseHeader + displayName: "Check License" + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: | + set -ev + + # prepare and activate virtualenv + python -m venv env/ + + chmod +x ./env/bin/activate + source ./env/bin/activate + + # clone azure-cli + git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install --upgrade pip + pip install -q azdev + + azdev setup -c ../azure-cli -r ./ + + azdev --version + az --version + + azdev verify license + + - job: StaticAnalysis + displayName: "Static Analysis" + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: pip install wheel==0.30.0 pylint==1.9.5 flake8==3.5.0 requests + displayName: 'Install wheel, pylint, flake8, requests' + - bash: python scripts/ci/source_code_static_analysis.py + displayName: "Static Analysis" + + - job: IndexVerify + displayName: "Verify Extensions Index" + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: 3.7 + - bash: | + #!/usr/bin/env bash + set -ev + pip install wheel==0.30.0 requests packaging + export CI="ADO" + python ./scripts/ci/test_index.py -v + displayName: "Verify Extensions Index" + + - job: SourceTests + displayName: "Integration Tests, Build Tests" + pool: + vmImage: 'ubuntu-latest' + strategy: + matrix: + Python36: + python.version: '3.6' + Python38: + python.version: '3.8' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install wheel==0.30.0 + displayName: 'Install wheel==0.30.0' + - bash: ./scripts/ci/test_source.sh + displayName: 'Run integration test and build test' + env: + ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId) + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) + + - job: LintModifiedExtensions + displayName: "CLI Linter on Modified Extensions" + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: | + set -ev + + # prepare and activate virtualenv + pip install virtualenv + python -m virtualenv venv/ + source ./venv/bin/activate + + # clone azure-cli + git clone --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install --upgrade pip + pip install azdev + + azdev --version + + azdev setup -c ../azure-cli -r ./ -e k8s-extension + + # overwrite the default AZURE_EXTENSION_DIR set by ADO + AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version + + AZURE_EXTENSION_DIR=~/.azure/cliextensions azdev linter --include-whl-extensions k8s-extension + displayName: "CLI Linter on Modified Extension" + env: + ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId) ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) \ No newline at end of file From d3d28c21fa9a716e200bda5bc358043a430893ba Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 26 Oct 2021 12:40:54 -0700 Subject: [PATCH 3/4] Remove the parallel invoke expression calls --- testing/Test.ps1 | 2 +- .../private-preview/AzurePolicy.Tests.ps1 | 36 +++++------ .../extensions/public/AzureDefender.Tests.ps1 | 30 +++++----- .../public/AzureMLKubernetes.Tests.ps1 | 60 +++++++++---------- .../extensions/public/AzureMonitor.Tests.ps1 | 28 ++++----- .../extensions/public/Cassandra.Tests.ps1 | 42 ++++++------- .../public/OpenServiceMesh.Tests.ps1 | 28 ++++----- 7 files changed, 113 insertions(+), 113 deletions(-) diff --git a/testing/Test.ps1 b/testing/Test.ps1 index a37287c4c33..08d8d080e7c 100644 --- a/testing/Test.ps1 +++ b/testing/Test.ps1 @@ -117,7 +117,7 @@ if ($ParallelCI) { } elseif ($CI) { Write-Host "Invoking Pester to run tests from '$testFilePath'..." $testResult = Invoke-Pester $testFilePath -Passthru -Output Detailed - $testResult | Export-JUnitReport -Path TestResults.xml + $testResult | Export-JUnitReport -Path "$testFileDirectory/TestResults.xml" } else { if ($Path) { Write-Host "Invoking Pester to run tests from '$PSScriptRoot/$Path'" diff --git a/testing/test/extensions/private-preview/AzurePolicy.Tests.ps1 b/testing/test/extensions/private-preview/AzurePolicy.Tests.ps1 index 467a4d7dd3c..bd0e0d75083 100644 --- a/testing/test/extensions/private-preview/AzurePolicy.Tests.ps1 +++ b/testing/test/extensions/private-preview/AzurePolicy.Tests.ps1 @@ -10,11 +10,11 @@ Describe 'Azure Policy Testing' { } It 'Creates the extension and checks that it onboards correctly' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait + $? | Should -BeTrue - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue @@ -35,17 +35,17 @@ Describe 'Azure Policy Testing' { } It "Performs a show on the extension" { - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty } It "Runs an update on the extension on the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName update -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --auto-upgrade false --no-wait" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName update -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --auto-upgrade false --no-wait + $? | Should -BeTrue - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion $isAutoUpgradeMinorVersion.ToString() -eq "False" | Should -BeTrue @@ -65,8 +65,8 @@ Describe 'Azure Policy Testing' { } It "Lists the extensions on the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } @@ -74,18 +74,18 @@ Describe 'Azure Policy Testing' { } It "Deletes the extension from the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue # Extension should not be found on the cluster - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -Not -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse $output | Should -BeNullOrEmpty } It "Performs another list after the delete" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } diff --git a/testing/test/extensions/public/AzureDefender.Tests.ps1 b/testing/test/extensions/public/AzureDefender.Tests.ps1 index 60d34932926..419d226cb2e 100644 --- a/testing/test/extensions/public/AzureDefender.Tests.ps1 +++ b/testing/test/extensions/public/AzureDefender.Tests.ps1 @@ -9,11 +9,11 @@ Describe 'Azure Defender Testing' { } It 'Creates the extension and checks that it onboards correctly' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait + $? | Should -BeTrue - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue @@ -23,7 +23,7 @@ Describe 'Azure Defender Testing' { do { # Only check the extension config, not the pod since this doesn't bring up pods - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName $provisioningState = ($output | ConvertFrom-Json).provisioningState Write-Host "Got ProvisioningState: $provisioningState for the extension" if (Has-ExtensionData $extensionName) { @@ -36,14 +36,14 @@ Describe 'Azure Defender Testing' { } It "Performs a show on the extension" { - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty } It "Lists the extensions on the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } @@ -51,18 +51,18 @@ Describe 'Azure Defender Testing' { } It "Deletes the extension from the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue # Extension should not be found on the cluster - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -Not -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse $output | Should -BeNullOrEmpty } It "Performs another list after the delete" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 index 81b1ee847d4..b1e4b3d1d39 100644 --- a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -13,11 +13,11 @@ Describe 'AzureML Kubernetes Testing' { It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { $sslKeyPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_key.pem" $sslCertPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_cert.pem" - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net inferenceLoadBalancerHA=False --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile --no-wait" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net inferenceLoadBalancerHA=False --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile --no-wait + $? | Should -BeTrue - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue @@ -40,15 +40,15 @@ Describe 'AzureML Kubernetes Testing' { } It "Performs a show on the extension" { - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty } It "Lists the extensions on the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } @@ -64,18 +64,18 @@ Describe 'AzureML Kubernetes Testing' { az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName - $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue # Extension should not be found on the cluster - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -Not -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse $output | Should -BeNullOrEmpty } It "Performs another list after the delete" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } @@ -83,11 +83,11 @@ Describe 'AzureML Kubernetes Testing' { } # It 'Creates the extension and checks that it onboards correctly with training enabled' { - # Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableTraining=true" -ErrorVariable badOut - # $badOut | Should -BeNullOrEmpty + # az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableTraining=true + # $? | Should -BeTrue - # $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - # $badOut | Should -BeNullOrEmpty + # $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeTrue # $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion # $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue @@ -118,21 +118,21 @@ Describe 'AzureML Kubernetes Testing' { # az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName # az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName - # $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - # $badOut | Should -BeNullOrEmpty + # $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeTrue # # Extension should not be found on the cluster - # $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - # $badOut | Should -Not -BeNullOrEmpty + # $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeFalse # $output | Should -BeNullOrEmpty # } # It 'Creates the extension and checks that it onboards correctly with inference enabled' { - # Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True inferenceLoadBalancerHA=false" -ErrorVariable badOut - # $badOut | Should -BeNullOrEmpty + # az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True inferenceLoadBalancerHA=false + # $? | Should -BeTrue - # $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - # $badOut | Should -BeNullOrEmpty + # $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeTrue # $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion # $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue @@ -163,12 +163,12 @@ Describe 'AzureML Kubernetes Testing' { # az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName # az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName - # $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - # $badOut | Should -BeNullOrEmpty + # $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeTrue # # Extension should not be found on the cluster - # $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - # $badOut | Should -Not -BeNullOrEmpty + # $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeFalse # $output | Should -BeNullOrEmpty # } } diff --git a/testing/test/extensions/public/AzureMonitor.Tests.ps1 b/testing/test/extensions/public/AzureMonitor.Tests.ps1 index 47ed96456ec..9748755eea0 100644 --- a/testing/test/extensions/public/AzureMonitor.Tests.ps1 +++ b/testing/test/extensions/public/AzureMonitor.Tests.ps1 @@ -10,11 +10,11 @@ Describe 'Azure Monitor Testing' { } It 'Creates the extension and checks that it onboards correctly' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait + $? | Should -BeTrue - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue @@ -33,14 +33,14 @@ Describe 'Azure Monitor Testing' { } It "Performs a show on the extension" { - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty } It "Lists the extensions on the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } @@ -48,18 +48,18 @@ Describe 'Azure Monitor Testing' { } It "Deletes the extension from the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue # Extension should not be found on the cluster - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -Not -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse $output | Should -BeNullOrEmpty } It "Performs another list after the delete" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } diff --git a/testing/test/extensions/public/Cassandra.Tests.ps1 b/testing/test/extensions/public/Cassandra.Tests.ps1 index 2261e241cf3..e579c43ee30 100644 --- a/testing/test/extensions/public/Cassandra.Tests.ps1 +++ b/testing/test/extensions/public/Cassandra.Tests.ps1 @@ -8,11 +8,11 @@ Describe 'Cassandra Testing' { } It 'Creates the extension and checks that it onboards correctly' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait + $? | Should -BeTrue - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue @@ -26,7 +26,7 @@ Describe 'Cassandra Testing' { do { # Only check the extension config, not the pod since this doesn't bring up pods - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName $provisioningState = ($output | ConvertFrom-Json).provisioningState Write-Host "Got ProvisioningState: $provisioningState for the extension" if ((Has-ExtensionData $extensionName) -And ($provisioningState -eq "Succeeded")) { @@ -34,22 +34,22 @@ Describe 'Cassandra Testing' { } Start-Sleep -Seconds 10 $n += 1 - } while ($n -le $MAX_RETRY_ATTEMPTS) - $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } while ($n -le 20) + $n | Should -BeLessOrEqual 20 } It "Performs a show on the extension" { - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty } It "Runs an update on the extension on the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName update -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --auto-upgrade false --no-wait" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName update -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --auto-upgrade false --no-wait + $? | Should -BeTrue - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion $isAutoUpgradeMinorVersion.ToString() -eq "False" | Should -BeTrue @@ -69,8 +69,8 @@ Describe 'Cassandra Testing' { } It "Lists the extensions on the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } @@ -78,18 +78,18 @@ Describe 'Cassandra Testing' { } It "Deletes the extension from the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue # Extension should not be found on the cluster - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -Not -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse $output | Should -BeNullOrEmpty } It "Performs another list after the delete" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } diff --git a/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 b/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 index dc446dede8b..6d14ccfce6c 100644 --- a/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 +++ b/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 @@ -14,11 +14,11 @@ Describe 'Azure OpenServiceMesh Testing' { # Should Not BeNullOrEmpty checks if the command returns JSON output It 'Creates the extension and checks that it onboards correctly' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train $releaseTrain --version $extensionVersion --no-wait" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train $releaseTrain --version $extensionVersion --no-wait + $? | Should -BeTrue - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion $isAutoUpgradeMinorVersion.ToString() -eq "False" | Should -BeTrue @@ -37,14 +37,14 @@ Describe 'Azure OpenServiceMesh Testing' { } It "Performs a show on the extension" { - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty } It "Lists the extensions on the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } @@ -52,18 +52,18 @@ Describe 'Azure OpenServiceMesh Testing' { } It "Deletes the extension from the cluster" { - $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue # Extension should not be found on the cluster - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -Not -BeNullOrEmpty + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse $output | Should -BeNullOrEmpty } It "Performs another list after the delete" { - $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue $output | Should -Not -BeNullOrEmpty $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } From 297037464a080aacb3c42dd865b6010e4c9d77a2 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 9 Nov 2021 10:42:48 -0800 Subject: [PATCH 4/4] Add templates to testing --- testing/Test.ps1 | 6 +- testing/pipeline/k8s-custom-pipelines.yml | 156 ++++------------------ testing/pipeline/templates/run-test.yml | 122 +++++++++++++++++ 3 files changed, 151 insertions(+), 133 deletions(-) create mode 100644 testing/pipeline/templates/run-test.yml diff --git a/testing/Test.ps1 b/testing/Test.ps1 index 08d8d080e7c..c1d4f093b2e 100644 --- a/testing/Test.ps1 +++ b/testing/Test.ps1 @@ -115,9 +115,13 @@ if ($ParallelCI) { throw "One or more tests failed" } } elseif ($CI) { + if ($Path) { + $testFilePath = "$PSScriptRoot/$Path" + } Write-Host "Invoking Pester to run tests from '$testFilePath'..." $testResult = Invoke-Pester $testFilePath -Passthru -Output Detailed - $testResult | Export-JUnitReport -Path "$testFileDirectory/TestResults.xml" + $testName = Split-Path $testFilePath –leaf + $testResult | Export-JUnitReport -Path "$testFileDirectory/$testName.xml" } else { if ($Path) { Write-Host "Invoking Pester to run tests from '$PSScriptRoot/$Path'" diff --git a/testing/pipeline/k8s-custom-pipelines.yml b/testing/pipeline/k8s-custom-pipelines.yml index c22ef78f122..77fcf7622d9 100644 --- a/testing/pipeline/k8s-custom-pipelines.yml +++ b/testing/pipeline/k8s-custom-pipelines.yml @@ -24,138 +24,30 @@ stages: EXTENSION_NAME: "k8s-extension" EXTENSION_FILE_NAME: "k8s_extension" jobs: - - job: K8sExtensionTestSuite - displayName: "Run the Test Suite" - pool: - vmImage: 'ubuntu-latest' - steps: - - checkout: self - - bash: | - echo "Installing helm3" - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 get_helm.sh - ./get_helm.sh --version v3.6.3 - - echo "Installing kubectl" - curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" - chmod +x ./kubectl - sudo mv ./kubectl /usr/local/bin/kubectl - kubectl version --client - displayName: "Setup the VM with helm3 and kubectl" - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: | - set -ev - echo "Building extension ${EXTENSION_NAME}..." - - # prepare and activate virtualenv - pip install virtualenv - python3 -m venv env/ - source env/bin/activate - - # clone azure-cli - git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli - - pip install --upgrade pip - pip install -q azdev - - ls $(CLI_REPO_PATH) - - azdev --version - azdev setup -c ../azure-cli -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME) - azdev extension build $(EXTENSION_NAME) - workingDirectory: $(CLI_REPO_PATH) - displayName: "Setup and Build Extension with azdev" - - - bash: | - K8S_EXTENSION_VERSION=$(ls ${EXTENSION_FILE_NAME}* | cut -d "-" -f2) - echo "##vso[task.setvariable variable=K8S_EXTENSION_VERSION]$K8S_EXTENSION_VERSION" - cp * $(TEST_PATH)/bin - workingDirectory: $(CLI_REPO_PATH)/dist - displayName: "Copy the Built .whl to Extension Test Path" - - - bash: | - RAND_STR=$RANDOM - AKS_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-aks" - ARC_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-arc" - - JSON_STRING=$(jq -n \ - --arg SUB_ID "$SUBSCRIPTION_ID" \ - --arg RG "$RESOURCE_GROUP" \ - --arg AKS_CLUSTER_NAME "$AKS_CLUSTER_NAME" \ - --arg ARC_CLUSTER_NAME "$ARC_CLUSTER_NAME" \ - --arg K8S_EXTENSION_VERSION "$K8S_EXTENSION_VERSION" \ - '{subscriptionId: $SUB_ID, resourceGroup: $RG, aksClusterName: $AKS_CLUSTER_NAME, arcClusterName: $ARC_CLUSTER_NAME, extensionVersion: {"k8s-extension": $K8S_EXTENSION_VERSION, connectedk8s: "1.0.0"}}') - echo $JSON_STRING > settings.json - cat settings.json - workingDirectory: $(TEST_PATH) - displayName: "Generate a settings.json file" - - - bash : | - echo "Downloading the kind script" - curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 - chmod +x ./kind - ./kind create cluster - displayName: "Create and Start the Kind cluster" - - - bash: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - displayName: "Upgrade az to latest version" - - - task: AzureCLI@2 - displayName: Bootstrap - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Bootstrap.ps1 -CI - workingDirectory: $(TEST_PATH) - - - task: AzureCLI@2 - displayName: Run the Test Suite Public Extensions Only - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Test.ps1 -CI -OnlyPublicTests -Type k8s-extension - workingDirectory: $(TEST_PATH) - continueOnError: true - condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'False')) - - - task: AzureCLI@2 - displayName: Run the Test Suite on Private + Public Extensions - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Test.ps1 -CI -Type k8s-extension - workingDirectory: $(TEST_PATH) - continueOnError: true - condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'True')) - - - task: PublishTestResults@2 - inputs: - testResultsFormat: 'JUnit' - testResultsFiles: '**/testing/test/results/*.xml' - failTaskOnFailedTests: true - condition: succeededOrFailed() - - - task: AzureCLI@2 - displayName: Cleanup - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Cleanup.ps1 -CI - workingDirectory: $(TEST_PATH) - condition: succeededOrFailed() - + - template: ./templates/run-test.yml + parameters: + jobName: AzureDefender + path: ./test/extensions/public/AzureDefender.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: AzureMLKubernetes + path: ./test/extensions/public/AzureMLKubernetes.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: AzureMonitor + path: ./test/extensions/public/AzureMonitor.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: AzurePolicy + path: ./test/extensions/public/AzurePolicy.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: Cassandra + path: ./test/extensions/public/Cassandra.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: OpenServiceMesh + path: ./test/extensions/public/OpenServiceMesh.Tests.ps1 - job: BuildPublishExtension pool: vmImage: 'ubuntu-latest' diff --git a/testing/pipeline/templates/run-test.yml b/testing/pipeline/templates/run-test.yml new file mode 100644 index 00000000000..5e3f7823c6a --- /dev/null +++ b/testing/pipeline/templates/run-test.yml @@ -0,0 +1,122 @@ +parameters: + jobName: '' + path: '' + +jobs: +- job: ${{ parameters.jobName}} + pool: + vmImage: 'ubuntu-latest' + steps: + - bash: | + echo "Installing helm3" + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh --version v3.6.3 + + echo "Installing kubectl" + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + kubectl version --client + displayName: "Setup the VM with helm3 and kubectl" + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: | + set -ev + echo "Building extension ${EXTENSION_NAME}..." + + # prepare and activate virtualenv + pip install virtualenv + python3 -m venv env/ + source env/bin/activate + + # clone azure-cli + git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install --upgrade pip + pip install -q azdev + + ls $(CLI_REPO_PATH) + + azdev --version + azdev setup -c ../azure-cli -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME) + azdev extension build $(EXTENSION_NAME) + workingDirectory: $(CLI_REPO_PATH) + displayName: "Setup and Build Extension with azdev" + + - bash: | + K8S_EXTENSION_VERSION=$(ls ${EXTENSION_FILE_NAME}* | cut -d "-" -f2) + echo "##vso[task.setvariable variable=K8S_EXTENSION_VERSION]$K8S_EXTENSION_VERSION" + cp * $(TEST_PATH)/bin + workingDirectory: $(CLI_REPO_PATH)/dist + displayName: "Copy the Built .whl to Extension Test Path" + + - bash: | + RAND_STR=$RANDOM + AKS_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-aks" + ARC_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-arc" + + JSON_STRING=$(jq -n \ + --arg SUB_ID "$SUBSCRIPTION_ID" \ + --arg RG "$RESOURCE_GROUP" \ + --arg AKS_CLUSTER_NAME "$AKS_CLUSTER_NAME" \ + --arg ARC_CLUSTER_NAME "$ARC_CLUSTER_NAME" \ + --arg K8S_EXTENSION_VERSION "$K8S_EXTENSION_VERSION" \ + '{subscriptionId: $SUB_ID, resourceGroup: $RG, aksClusterName: $AKS_CLUSTER_NAME, arcClusterName: $ARC_CLUSTER_NAME, extensionVersion: {"k8s-extension": $K8S_EXTENSION_VERSION, connectedk8s: "1.0.0"}}') + echo $JSON_STRING > settings.json + cat settings.json + workingDirectory: $(TEST_PATH) + displayName: "Generate a settings.json file" + + - bash : | + echo "Downloading the kind script" + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 + chmod +x ./kind + ./kind create cluster + displayName: "Create and Start the Kind cluster" + + - bash: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + displayName: "Upgrade az to latest version" + + - task: AzureCLI@2 + displayName: Bootstrap + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Bootstrap.ps1 -CI + workingDirectory: $(TEST_PATH) + + - task: AzureCLI@2 + displayName: Run the Test Suite for ${{ parameters.path }} + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Test.ps1 -CI -Path ${{ parameters.path }} -Type k8s-extension + workingDirectory: $(TEST_PATH) + continueOnError: true + condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'False')) + + - task: PublishTestResults@2 + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '**/testing/results/*.xml' + failTaskOnFailedTests: true + condition: succeededOrFailed() + + - task: AzureCLI@2 + displayName: Cleanup + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Cleanup.ps1 -CI + workingDirectory: $(TEST_PATH) + condition: succeededOrFailed() \ No newline at end of file