Merge pull request #7 from Azure/main

Update 7/27

doc/dev/perfstress_tests.md

@@ -2,7 +2,7 @@
 1. [The perfstress framework](#the-perfstress-framework)
     - [The PerfStressTest base](#the-perfstresstest-base)
     - [Default command options](#default-command-options)
-    - [Running with test proxy](#running-with-test-proxy)
+    - [Running with test proxy](#running-with-the-test-proxy)
 2. [Adding performance tests to an SDK](#adding-performance-tests-to-an-sdk)
     - [Writing a test](#writing-a-test)
     - [Adding legacy T1 tests](#adding-legacy-t1-tests)
@@ -88,7 +88,7 @@ The framework has a series of common command line options built in:
 
 ## Running with the test proxy
 Follow the instructions here to install and run the test proxy server:
-https://github.com/Azure/azure-sdk-tools/tree/feature/http-recording-server/tools/test-proxy/Azure.Sdk.Tools.TestProxy
+https://github.com/Azure/azure-sdk-tools/tree/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy
 
 Once running, in a separate process run the perf test in question, combined with the `-x` flag to specify the proxy endpoint.
 ```cmd
@@ -375,4 +375,4 @@ Using the `perfstress` command alone will list the available perf tests found. N
 Please add a `README.md` to the perfstress_tests directory so that others know how to setup and run the perf tests, along with a description of the available tests and any support command line options. README files in a `tests/perfstress_tests` directory should already be filtered from CI validation for SDK readmes.
 Some examples can be found here:
 - [Azure Storage Blob](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/storage/azure-storage-blob/tests/perfstress_tests/README.md)
-- [Azure Service Bus](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/servicebus/azure-servicebus/tests/perf_tests/README.md)
+- [Azure Service Bus](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/servicebus/azure-servicebus/tests/perf_tests/README.md)

eng/ci_tools.txt

@@ -18,7 +18,7 @@ coverage==4.5.4
 codecov==2.1.0
 beautifulsoup4==4.9.1
 pkginfo==1.5.0.1
-pip==20.2
+pip==20.3
 black==21.6b0; python_version >= '3.6'
 
 # locking packages defined as deps from azure-sdk-tools or azure-devtools

eng/common/pipelines/templates/steps/bypass-local-dns.yml

@@ -6,11 +6,6 @@ steps:
       condition: |
         and(
           succeededOrFailed(),
-          or(
-            eq(variables['OSVmImage'], 'ubuntu-18.04'),
-            eq(variables['OSVmImage'], 'ubuntu-20.04'),
-            eq(variables['OSVmImage'], 'MMSUbuntu18.04'),
-            eq(variables['OSVmImage'], 'MMSUbuntu20.04')
-          ),
+          contains(variables['OSVmImage'], 'ubuntu'),
           eq(variables['Container'], '')
         )

eng/common/scripts/FilterPoliCheckResults.ps1

@@ -0,0 +1,103 @@
+<#
+.SYNOPSIS
+Filters PoliCheck Result.
+.DESCRIPTION
+This script will read data speciefied in one or more PoliCheckAllowList.yml files,
+It then reamoves all allwed entries from the PoliCheckResult 
+.PARAMETER PoliCheckResultFilePath
+The Path to the PoliCheck Result. Usually named PoliCheck.sarif
+.PARAMETER ServiceDirtectory
+If the PoliCheck scan is scoped to a particular service provide the ServiceDirectory
+.EXAMPLE
+PS> ./FilterPoliCheckResults.ps1 -PoliCheckResultFilePath .\PoliCheck.sarif
+#>
+[CmdletBinding()]
+param(
+  [Parameter(Mandatory=$true)]
+  [String] $PoliCheckResultFilePath,
+  [String] $ServiceDirtectory
+)
+
+. "${PSScriptRoot}\logging.ps1"
+
+$RepoRoot = Resolve-Path -Path "${PSScriptRoot}\..\..\..\"
+$PathToAllowListFiles = Join-Path $RepoRoot $ServiceDirtectory
+$PolicCheckAllowListFiles = Get-ChildItem -Path $PathToAllowListFiles -Recurse -File -Include "PoliCheckAllowList.yml"
+$allowListData = @{}
+
+# Combine all AllowLists Found
+foreach ($file in $PolicCheckAllowListFiles)
+{
+    $allowListDataInFile = ConvertFrom-Yaml (Get-Content $file.FullName -Raw)
+    $allowListData["PC1001"] += $allowListDataInFile["PC1001"]
+    $allowListData["PC1002"] += $allowListDataInFile["PC1002"]
+    $allowListData["PC1003"] += $allowListDataInFile["PC1003"]
+    $allowListData["PC1004"] += $allowListDataInFile["PC1004"]
+    $allowListData["PC1005"] += $allowListDataInFile["PC1005"]
+    $allowListData["PC1006"] += $allowListDataInFile["PC1006"]
+}
+
+$poliCheckData = Get-Content $PoliCheckResultFilePath | ConvertFrom-Json
+$poliCheckResultsCount = $poliCheckData.runs[0].results.Count
+$newCount
+
+$updatedRuns = @()
+
+foreach ($run in $poliCheckData.runs)
+{
+    $updatedResults = @()
+    foreach ($result in $run.results)
+    {
+        $ruleId = $result.ruleId
+        $allowedEntries = $allowListData[$ruleId]
+        if ($allowedEntries)
+        {
+            $updatedLocations = @()
+
+            foreach ($location in $result.locations)
+            {
+                $filePath = $location.physicalLocation.artifactLocation.uri
+                $text = $location.physicalLocation.region.snippet.text
+                $contextRegion = $location.physicalLocation.contextRegion.snippet.text
+
+                $allowedEntry = $allowedEntries[0] | Where-Object { $_.FilePath -eq $filePath }
+
+                if ($allowedEntry.Count -gt 0)
+                {
+                    $foundAllowedInstance = $false
+                    foreach ($instance in $allowedEntry.instances)
+                    {
+                        if (($instance.Text.Trim() -eq $text.Trim()) -and ($instance.ContextRegion.Trim() -eq $contextRegion.Trim()))
+                        {
+                            Write-Host "Found instance" -ForegroundColor Green
+                            $foundAllowedInstance = $true
+                        }
+                    }
+                    if ($foundAllowedInstance -eq $true)
+                    {
+                        continue
+                    }
+                }
+
+                $updatedLocations += $location
+            }
+
+            $result.locations = $updatedLocations
+        }
+
+        if ($result.locations.Count -gt 0)
+        {
+            $updatedResults += $result 
+        }
+    }
+    $run.results = $updatedResults
+    $newCount = $run.results.Count
+    $updatedRuns += $run
+}
+
+$poliCheckData.runs = $updatedRuns
+
+Set-Content -Path $PoliCheckResultFilePath -Value ($poliCheckData | ConvertTo-Json -Depth 100)
+
+LogDebug "Original Result Count: ${poliCheckResultsCount}"
+LogDebug "New Result Count: ${newCount}"

eng/common/scripts/Update-DocsMsMetadata.ps1

@@ -84,7 +84,7 @@ keywords: Azure, $Language, SDK, API, $($PackageInfo.Name), $service
 author: maggiepint
 ms.author: magpint
 ms.date: $date
-ms.topic: article
+ms.topic: reference
 ms.prod: azure
 ms.technology: azure
 ms.devlang: $Language

eng/common/scripts/update-docs-metadata.ps1

@@ -71,7 +71,7 @@ function GetAdjustedReadmeContent($pkgInfo){
     $ReplacementPattern = "`${1}$($pkgInfo.Tag)"
     $fileContent = $fileContent -replace $releaseReplaceRegex, $ReplacementPattern
 
-    $header = "---`ntitle: $foundTitle`nkeywords: Azure, $Language, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $Language`nms.service: $service`n---`n"
+    $header = "---`ntitle: $foundTitle`nkeywords: Azure, $Language, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: reference`nms.prod: azure`nms.technology: azure`nms.devlang: $Language`nms.service: $service`n---`n"
 
     if ($fileContent) {
       return "$header`n$fileContent"

eng/pipelines/autorest_checks.yml

@@ -40,25 +40,24 @@ jobs:
           versionSpec: $(PythonVersion)
 
       - script: |
-          pip install -r eng/autorest_req.txt
           git clone $(auto_rest_clone_url)
           cd autorest.python
           git checkout $(repo_branch)
           npm install
         displayName: 'Prepare Environment'
 
       - script: |
-          cd $(Build.SourcesDirectory)/autorest.python/test/vanilla
+          cd $(Build.SourcesDirectory)/autorest.python/test/vanilla/legacy
           pip install $(Build.SourcesDirectory)/$(source_path_azure_core)
           pip install -r requirements.txt
           pip freeze
-          pytest $(Build.SourcesDirectory)/autorest.python/test/vanilla
-        displayName: 'Install azure-core and Test Vanilla'
+          pytest $(Build.SourcesDirectory)/autorest.python/test/vanilla/legacy
+        displayName: 'Install azure-core and Test Vanilla Legacy'
 
       - script: |
-          cd $(Build.SourcesDirectory)/autorest.python/test/azure
+          cd $(Build.SourcesDirectory)/autorest.python/test/azure/legacy
           pip install $(Build.SourcesDirectory)/$(source_path_azure_mgmt_core)
           pip install -r requirements.txt
           pip freeze
-          pytest $(Build.SourcesDirectory)/autorest.python/test/azure
-        displayName: 'Install azure-mgmt-core and Test Azure'
+          pytest $(Build.SourcesDirectory)/autorest.python/test/azure/legacy
+        displayName: 'Install azure-mgmt-core and Test Azure Legacy'

eng/pipelines/templates/jobs/tests-nightly-python.yml

@@ -38,8 +38,10 @@ jobs:
           python3 -m pip install tox tox-monorepo packaging twine codecov beautifulsoup4
           python3 --version
           cd $(Build.SourcesDirectory)
-          python3 ./scripts/devops_tasks/setup_execute_tests.py "$(BuildTargetingString)" --junitxml="junit/test_results_38.xml" --toxenv="whl,sdist"
+          python3 ./scripts/devops_tasks/setup_execute_tests.py "$(BuildTargetingString)" --junitxml="junit/test_results_38.xml" --toxenv="whl"
         displayName: 'Setup - Run Filtered Tests "Nightly" using Python Edge'
+        env:
+          YARL_NO_EXTENSIONS: 1
         continueOnError: true
 
       - task: PublishTestResults@2

eng/pipelines/templates/steps/build-test.yml

@@ -90,7 +90,7 @@ steps:
 
   - task: PythonScript@0
     displayName: 'Test Samples'
-    condition: eq(variables['TestSamples'], 'true')
+    condition: and(succeeded(), eq(variables['TestSamples'], 'true'))
     inputs:
       scriptPath: 'scripts/devops_tasks/setup_execute_tests.py'
       arguments: >-

eng/pipelines/templates/steps/verify-autorest.yml

@@ -32,23 +32,6 @@ steps:
         autorest --help
       displayName: "Install autorest"
 
-    - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-      parameters:
-        Paths:
-          - "/*"
-        Repositories:
-          - Name: "Azure/autorest.python"
-            Commitish: "master"
-            WorkingDirectory: "$(Build.SourcesDirectory)/autorest.python"
-        SkipDefaultCheckout: true
-
-    - script: |
-        python --version
-        pip install -r eng/autorest_req.txt
-        cd autorest.python
-        npm install
-      displayName: 'Prepare Environment'
-
     - task: PythonScript@0
       displayName: 'Verify autorest'
       inputs:

eng/test_tools.txt

@@ -1,4 +1,4 @@
-pip==20.2
+pip==20.3
 
 # requirements leveraged by ci for testing
 pytest==4.6.9; python_version == '2.7'

eng/tox/install_depend_packages.py

@@ -13,6 +13,7 @@
 import logging
 from packaging.specifiers import SpecifierSet
 from pkg_resources import Requirement, parse_version
+
 from pypi_tools.pypi import PyPIClient
 
 setup_parser_path = os.path.abspath(
@@ -30,11 +31,12 @@
 MINIMUM_VERSION_SUPPORTED_OVERRIDE = {
     'azure-common': '1.1.10',
     'msrest': '0.6.10',
-    'six': '1.9',
     'typing-extensions': '3.6.5',
     'opentelemetry-api': '1.3.0',
-    'cryptography': '3.3',
+    'opentelemetry-sdk': '1.3.0',
     'azure-core': '1.11.0',
+    'requests': '2.19.0',
+    'six': '1.12.0'
 }
 
 def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir):

eng/tox/sanitize_setup.py

@@ -96,7 +96,7 @@ def process_requires(setup_py_path):
     requires = [
         Requirement.parse(r)
         for r in get_install_requires(setup_py_path)
-        if r.startswith("azure") and "-nspkg" not in r
+        if r.startswith("azure")
     ]
     # Find package requirements that are not available on PyPI
     requirement_to_update = {}

eng/versioning/version_increment.py

@@ -42,7 +42,7 @@ def increment_version(old_version):
 
     package_name = args.package_name.replace('_', '-')
 
-    packages = get_packages(args, package_name)
+    packages = get_packages(args, package_name, additional_excludes = ["mgmt", "-nspkg"])
 
     package_map = { pkg[1][0]: pkg for pkg in packages }
 

eng/versioning/version_shared.py

@@ -34,8 +34,8 @@
 
 logging.getLogger().setLevel(logging.INFO)
 
-def path_excluded(path):
-    return "-nspkg" in path or "tests" in path or "mgmt" in path or is_metapackage(path)
+def path_excluded(path, additional_excludes):
+    return any([excl in path for excl in additional_excludes]) or "tests" in path or is_metapackage(path)
 
 # Metapackages do not have an 'azure' folder within them
 def is_metapackage(package_path):
@@ -44,13 +44,13 @@ def is_metapackage(package_path):
     azure_path = path.join(dir_path, 'azure')
     return not path.exists(azure_path)
 
-def get_setup_py_paths(glob_string, base_path):
+def get_setup_py_paths(glob_string, base_path, additional_excludes):
     setup_paths = process_glob_string(glob_string, base_path)
-    filtered_paths = [path.join(p, 'setup.py') for p in setup_paths if not path_excluded(p)]
+    filtered_paths = [path.join(p, 'setup.py') for p in setup_paths if not path_excluded(p, additional_excludes)]
     return filtered_paths
 
 
-def get_packages(args, package_name = ""):
+def get_packages(args, package_name = "", additional_excludes = []):
     # This function returns list of path to setup.py and setup info like install requires, version for all packages discovered using glob
     # Followiong are the list of arguements expected and parsed by this method
     # service, glob_string
@@ -59,7 +59,7 @@ def get_packages(args, package_name = ""):
     else:
         target_dir = root_dir
 
-    paths = get_setup_py_paths(args.glob_string, target_dir)
+    paths = get_setup_py_paths(args.glob_string, target_dir, additional_excludes)
 
     # Check if package is excluded if a package name param is passed
     if package_name and not any(filter(lambda x: package_name == os.path.basename(os.path.dirname(x)), paths)):

scripts/auto_release/livetest_folder_link.py

@@ -121,4 +121,6 @@
   'timeseriesinsights' : 'timeseriesinsights',
   'trafficmanager' : 'trafficmanager',
   'workloadmonitor' : 'workloadmonitor',
+  'guestconfig' : 'machinelearning',
+  'avs' : 'compute',
 }

scripts/devops_tasks/git_helper.py

@@ -26,7 +26,10 @@
     'azure-schemaregistry-avroserializer': '1.0.0b1',
     'azure-storage-blob-changefeed' : '12.0.0b2',
     'azure-storage-file-datalake': '12.2.0b1',
-    'azure-communication-identity': '1.0.0'
+    'azure-communication-identity': '1.0.0',
+    'azure-communication-phonenumbers': '1.0.0',
+    'azure-communication-sms': '1.0.0',
+    'azure-search-documents': '11.2.0b3',
 }
 
 # This method identifies release tag for latest or oldest released version of a given package

sdk/attestation/azure-security-attestation/README.md

@@ -327,10 +327,10 @@ section of the project.
 [azure_identity]: https://docs.microsoft.com/python/api/overview/azure/identity-readme?view=azure-python-preview
 [DefaultAzureCredential]: https://docs.microsoft.com/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python
 [ClientSecretCredential]: https://docs.microsoft.com/python/api/azure-identity/azure.identity.clientsecretcredential?view=azure-python
-[attestation_policy_result]:https://docs.microsoft.com/python/api/azure-security-attestation/azure.security.attestation.policyresult?view=azure-python-preview
-[attestation_client]: https://docs.microsoft.com/python/api/azure-security-attestation/azure.security.attestation.attestationclient?view=azure-python-preview
-[attestation_admin_client]: https://docs.microsoft.com/python/api/azure-security-attestation/azure.security.attestation.attestationadministrationclient?view=azure-python-preview
-[attestation_policy_result_parameters]: https://docs.microsoft.com/python/api/azure-security-attestation/azure.security.attestation.policyresult?view=azure-python-preview#parameters
+[attestation_policy_result]:https://docs.microsoft.com/python/api/azure-security-attestation/azure.security.attestation.attestationpolicyresult
+[attestation_client]: https://docs.microsoft.com/python/api/azure-security-attestation/azure.security.attestation.attestationclient
+[attestation_admin_client]: https://docs.microsoft.com/python/api/azure-security-attestation/azure.security.attestation.attestationadministrationclient
+[attestation_policy_result_parameters]: https://docs.microsoft.com/python/api/azure-security-attestation/azure.security.attestation.attestationpolicyresult#parameters
 [attest_sgx]: https://docs.microsoft.com/python/api/azure-security-attestation/azure.security.attestation.attestationclient?view=azure-python-preview#attest-sgx-enclave-quote--inittime-data-none--runtime-data-none--draft-policy-none----kwargs-
 [attestation_pypi]: https://aka.ms/azsdk/python/azure-security-attestation
 [API_reference]:https://docs.microsoft.com/python/api/overview/azure/security-attestation-readme?view=azure-python-preview

sdk/batch/azure-mgmt-batch/dev_requirements.txt

@@ -1,4 +1,4 @@
 -e ../../../tools/azure-sdk-tools
--e ../../keyvault/azure-mgmt-keyvault
--e ../../network/azure-mgmt-network
+azure-mgmt-keyvault<9.0.0
+azure-mgmt-network<19.0.0
 -e ../../../tools/azure-devtools