Update 4/19

.azure-pipelines/templates/automation_test.yml

@@ -8,9 +8,13 @@ parameters:
   type: string
   default: latest
 - name: fullTest
-  displayName: 'Run full test?'
+  displayName: Run full test?
   type: boolean
   default: false
+- name: module
+  displayName: Run test for specific module
+  type: string
+  default: ''
 
 steps:
   - task: UsePythonVersion@0
@@ -26,15 +30,24 @@ steps:
       echo pythonVersion: ${{ parameters.pythonVersion }}
       echo profile: ${{ parameters.profile }}
       echo fullTest: ${{ parameters.fullTest }}
+      echo module: ${{ parameters.module }}
       echo Build.Reason: $(Build.Reason)
 
+      # Test specific module
+      module="${{ parameters.module }}"
+      if [[ -n $module ]]; then
+        echo "Running test for module '$module'"
+        azdev test --no-exitfirst --verbose --series $module
+        exit 0
+      fi
+
       if [[ "$(Build.Reason)" == "PullRequest" && "${{ parameters.fullTest }}" == 'False' ]]; then
         echo "Running incremental test"
         azdev test --no-exitfirst --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --profile ${{ parameters.profile }} --verbose --series
       else
         echo "Running full test"
         azdev test --no-exitfirst --profile ${{ parameters.profile }} --verbose --series
       fi
-    displayName: "Test on Profile ${{ parameters.profile }}"
+    displayName: "azdev test"
     env:
       ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.TargetBranch)

.azure-pipelines/templates/azdev_setup.yml

@@ -21,7 +21,7 @@ steps:
       else
         azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug
       fi
-    displayName: 'Azdev Setup'
+    displayName: 'azdev setup'
     env:
       CLI_REPO_PATH: ${{ parameters.CLIRepoPath }}
       CLI_EXT_REPO_PATH: ${{ parameters.CLIExtensionRepoPath }}

azure-pipelines.yml

@@ -189,7 +189,7 @@ jobs:
     inputs:
       TargetPath: '$(Build.ArtifactStagingDirectory)/metadata'
       artifactName: metadata
-  
+
   - task: DownloadPipelineArtifact@1
     displayName: 'Download Build Artifacts'
     inputs:
@@ -201,7 +201,7 @@ jobs:
     inputs:
       targetType: inline
       script: |
-        if (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) {  
+        if (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) {
           # Start another Powershell process as Admin and execute this script again
           $arguments = "& '" +$myinvocation.mycommand.definition + "'"
           Start-Process powershell -Verb runAs -ArgumentList $arguments
@@ -223,7 +223,7 @@ jobs:
         $to_be_installed_version=Get-Content $(System.ArtifactsDirectory)/metadata/version
         if ($pre_installed_version -eq $to_be_installed_version){
           # See https://docs.microsoft.com/windows/win32/msi/reinstallmode about options of REINSTALLMODE
-          $reinstall_option="REINSTALL=ALL REINSTALLMODE=emus" 
+          $reinstall_option="REINSTALL=ALL REINSTALLMODE=emus"
           $InstallArgs += $reinstall_option
         }
         Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow
@@ -372,31 +372,39 @@ jobs:
 
     displayName: 'Test pip Install'
 
-- job: UnitTest
-  displayName: Unit Test for Core and Telemetry
+- job: TestCore
+  displayName: Unit Test for Core
   timeoutInMinutes: 10
+  pool:
+    vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
+  steps:
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        module: 'azure-cli-core'
 
+- job: TestTelemetry
+  displayName: Unit Test for Telemetry
+  timeoutInMinutes: 10
   pool:
     vmImage: 'ubuntu-16.04'
   strategy:
     matrix:
       Python36:
         python.version: '3.6'
-        tox_env: 'py36'
       Python38:
         python.version: '3.8'
-        tox_env: 'py38'
   steps:
-    - task: UsePythonVersion@0
-      displayName: 'Use Python $(python.version)'
-      inputs:
-        versionSpec: '$(python.version)'
-    - bash: pip install --upgrade pip tox
-      displayName: 'Install pip and tox'
-    - bash: ./scripts/ci/unittest.sh
-      displayName: 'Run Unit Test'
-      env:
-        TOXENV: $(tox_env)
+    - template: .azure-pipelines/templates/automation_test.yml
+      parameters:
+        pythonVersion: '$(python.version)'
+        module: 'azure-cli-telemetry'
 
 - job: IntegrationTestAgainstProfiles
   displayName: Integration Test against Profiles
@@ -858,6 +866,7 @@ jobs:
   - bash: |
       set -ev
       . env/bin/activate
+      python scripts/ci/service_name.py
       if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then
         azdev linter --ci-exclusions --min-severity medium --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch)
       else

doc/track_2_migration_guidance.md

@@ -1,12 +1,17 @@
 # Track 2 Migration Guidance
 
-Azure CLI is built upon Azure Python SDK. Recently Azure Python SDK announced next generation product. It is named Track 2 SDK. The old version of SDK is named Track 1. It claims that it has great advantages than Track 1 SDK. It is not compatible with Track 1 SDK. Azure CLI developers need to spend considerable time and do some work to migrate from Track 1 SDK to Track 2 SDK. Let's see an example of Track 2 SDK. [azure-mgmt-compute 17.0.0b1](https://pypi.org/project/azure-mgmt-compute/17.0.0b1/) introduces important breaking changes and important new features like unified authentication and asynchronous programming.
+Azure CLI is built on Azure Python SDKs. Recently, Azure Python SDK team announced the next generation product, named Track 2 SDK. The old version of SDK is called Track 1 SDK. It claims that it has some advantages than Track 1 SDK. It is not compatible with Track 1 SDK. A considerable number of work days are required for Azure CLI developers to migrate their modules from Track 1 SDK to Track 2 SDK. Let's see an example of Track 2 SDK. [azure-mgmt-compute 17.0.0b1](https://pypi.org/project/azure-mgmt-compute/17.0.0b1/) introduces important breaking changes and important new features like unified authentication and asynchronous programming.
 
 This document summarizes typical issues and solutions when adopting Track 2 SDK in Azure CLI.
 
 Example PRs:
 1. [Compute PR #15750](https://github.com/Azure/azure-cli/pull/15750)
 2. [Network PR #16245](https://github.com/Azure/azure-cli/pull/16245)
+3. [Network PR #16350](https://github.com/Azure/azure-cli/pull/16350)
+4. [Storage PR #15845](https://github.com/Azure/azure-cli/pull/15845)
+5. [KeyVault PR #14150](https://github.com/Azure/azure-cli/pull/14150)
+6. [AppConfig PR #16376](https://github.com/Azure/azure-cli/pull/16376)
+7. [AppService PR #17146](https://github.com/Azure/azure-cli/pull/17146)
 
 Below is a list of typical issues.
 
@@ -26,11 +31,12 @@ Below is a list of typical issues.
 
 Long running operations have changed their function names in Track 2 SDK. A `begin_` prefix is added. For example, `create_or_update` becomes `begin_create_or_update`. `delete` becomes `begin_delete`. It is a naming convention in Track 2 SDK to indicate that an operation is a long running operation. Test cases can reveal most instances, but if a command has no test, it may be missed. A reliable approach is going through all methods to see whether they are long running operations.
 
+
 ### Property name change
 
 Some of property names change in Track 2 SDK.
 
-Examples:
+Here are some examples. `->` is not Python code. It represents the left one is updated to the right one.
 
 ```
 hyperVgeneration -> hyperVGeneration
@@ -40,6 +46,7 @@ virtual_machine_extension_type -> type_properties_type
 type1 -> type_properties_type
 instance_ids -> vm_instance_i_ds
 diskMbpsReadWrite -> diskMBpsReadWrite
+ipv4 -> I_PV4
 ```
 
 Some changes are unnecessary, even wrong in English. I opened [Azure/autorest.python#850](https://github.com/Azure/autorest.python/issues/850) to track this problem.
@@ -48,7 +55,7 @@ Some changes are unnecessary, even wrong in English. I opened [Azure/autorest.py
 
 Some of class names change in Track 2 SDK.
 
-Examples:
+[Example](https://github.com/Azure/azure-cli/pull/15750/files#diff-fd5160263d5431e9cdbf0f83abad213589c44c4c2724ff66b1172218caeb8396R629):
 
 ```
 VirtualMachineIdentityUserAssignedIdentitiesValue -> UserAssignedIdentitiesValue
@@ -61,7 +68,9 @@ Error type changes in Track2 SDK.
 Examples:
 
 ```
-CloudError -> azure.core.exceptions.ResourceNotFoundError
+msrestazure.azure_exceptions.CloudError -> azure.core.exceptions.ResourceNotFoundError
+msrest.exceptions.ClientException -> azure.core.exceptions.HttpResponseError
+ErrorException -> HttpResponseError
 ```
 
 ### No enum type
@@ -70,13 +79,24 @@ Track 2 SDK removes enum type and adopts string type instead. It loses the valid
 
 ### Class hierarchy change
 
-The class hierarchy may change in Track 2 SDK. Some properties are not flattened. They are wrapped in classes. 
+The class hierarchy may change in Track 2 SDK. Some properties are not flattened. They are wrapped in classes. In Track 1 SDK, if the number of parameters is less than 3, it will be flattened.
 
 Examples:
 
-In VMSS `begin_update_instances`, a new type `VirtualMachineScaleSetVMInstanceRequiredIDs` is added.
+In [VMSS](https://github.com/Azure/azure-cli/pull/15750/files?file-filters%5B%5D=.py#diff-fd5160263d5431e9cdbf0f83abad213589c44c4c2724ff66b1172218caeb8396R2688) `begin_update_instances`, a new type `VirtualMachineScaleSetVMInstanceRequiredIDs` is added.
+
+In [DiskAccess](https://github.com/Azure/azure-cli/pull/15750/files#diff-fd5160263d5431e9cdbf0f83abad213589c44c4c2724ff66b1172218caeb8396R3602) `begin_create_or_update`, location and tags are moved to a nested structure `DiskAccess`, `disk_access = DiskAccess(location=location, tags=tags)`
+
+In [Storage](https://github.com/Azure/azure-cli/pull/15845/files#diff-4cfe9a680ae04774e116b45bc06a679db751bfad1de211c6d2b3bc471900d8bfR23),
+```
+client.check_name_availability(account_name) // account_name is string
+```
+turns into
+```
+account_name = StorageAccountCheckNameAvailabilityParameters(name=name) client.check_name_availability(account_name)
+```
 
-In DiskAccess `begin_create_or_update`, location and tags are moved to a nested structure `DiskAccess`, `disk_access = DiskAccess(location=location, tags=tags)`
+In [AppConfig](https://github.com/Azure/azure-cli/pull/16376/files#diff-1796b5bb574aca9235e83b02a207cb8a42aafab920f3aae1c46af22bf0ce5aa4R191), `id` cannot be passed directly to `regenerate_key method`. It needs to be wrapped in the new model `RegenerateKeyParameters`.
 
 ### Obtaining subscription
 
@@ -104,6 +124,8 @@ The problem I met is property name change. It is hard to find the line of code t
 
 It is only used in CI jobs. It patches some code to SDK. This file should be deprecated. It was written long time ago. But for now, just modify this file and add missing packages.
 
+[Example](https://github.com/Azure/azure-cli/pull/15750/files#diff-e1256a3d1d91aea524b252fa7dc4a64b83d183b7f57fb5c326b270a1c4b224a7)
+
 ### Missing external tenant authentication support
 
 External tenant authentication support is missing in Azure CLI Core and Azure Core package. In request header, we use x-ms-authorization-auxiliary to pass auxiliary authorization token. Compute module is the first customer to have this requirement in Track 2 SDK. In azure/core/pipeline/policies/_authentication.py, there is a class BearerTokenCredentialPolicy. It simplifies bearer token authorization header configuration. However, auxiliary bearer token authorization is not supported in Azure Core policies [1] yet. The current solution is getting tokens manually, setting headers in client constructor or in operation call time manually. They said they will support this policy in the future. Azure CLI Core also needs an upgrade to provide better interfaces for Track 2 SDK users.

doc/track_2_migration_roadmap.md

@@ -33,7 +33,7 @@ This document provides the roadmap for Track 2 SDK migration in Azure CLI. When
 | KeyVault    | Yes|**Completed** | 04/30/2021 |Catherine||GA (8.0.0) | GA (2.2.0) | Medium (Once three months) | Module |
 | Monitor     |Yes |**Completed** | 04/30/2021 |Yong||Monitor: GA (2.0.0)<br> LogAnalytics: GA (8.0.0) | Monitor: Preview (0.12.0)<br> LogAnalytics: GA (2.0.0) | Medium (Once two months) | Module |
 | Resource   | Yes |**In progress** | 04/30/2021 |Yong ||GA (15.0.0) | GA (12.0.0) | Medium (Once two months) | Module |
-| AppService   |  Yes | **In progress** | 04/30/2021 |Catherine||GA (1.0.0) | Preview (0.48.0) | Medium (Once three months) | Module |
+| AppService   |  Yes | **Completed** | 04/30/2021 |Catherine||GA (1.0.0) | Preview (0.48.0) | Medium (Once three months) | Module |
 | ACS   |  Yes |Not Started | 04/30/2021 |Catherine||ContainerService: GA (14.0.0)<br> ContainerInstance: GA (7.0.0) | ContainerService: GA (11.0.0)<br> ContainerInstance: GA (2.0.0) | Medium (Once two months) | Module |
 | ACR   | Yes |Not Started | 04/30/2021 |Catherine||Preview (8.0.0b1) | Preview (3.0.0rc16) | Medium (Once two months) | Module |
 | CosmosDB   | Yes |**Completed** | 04/30/2021 |Yong||GA (6.0.0) | Preview (2.0.0rc1) |  Medium (Once two months) |Module|

scripts/ci/build.sh

@@ -155,6 +155,7 @@ cat >>$testsrc_dir/setup.py <<EOL
                        '**/*.pfx',
                        '**/*.txt',
                        '**/*.json',
+                       '**/*.ipynb',
                        '**/*.byok',
                        '**/*.js',
                        '**/*.md',

scripts/ci/service_name.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+"""
+Check format of service_name.json. Command and AzureServiceName are required. Others are optional.
+Each highest level command group should have reference in service_name.json.
+"""
+import json
+
+from azure.cli.core import MainCommandsLoader, AzCli
+from azure.cli.core._help import AzCliHelp
+from azure.cli.core.commands import AzCliCommandInvoker
+from azure.cli.core.file_util import create_invoker_and_load_cmds_and_args, get_all_help
+from azure.cli.core.parser import AzCliCommandParser
+
+
+def main():
+    az_cli = AzCli(cli_name='az',
+                   commands_loader_cls=MainCommandsLoader,
+                   invocation_cls=AzCliCommandInvoker,
+                   parser_cls=AzCliCommandParser,
+                   help_cls=AzCliHelp)
+    create_invoker_and_load_cmds_and_args(az_cli)
+    help_files = get_all_help(az_cli)
+    high_command_set = set()
+    for help_file in help_files:
+        if help_file.command:
+            high_command_set.add(help_file.command.split()[0])
+    print('high_command_set:')
+    print(high_command_set)
+
+    # Load and check service_name.json
+    with open('src/azure-cli/service_name.json') as f:
+        service_names = json.load(f)
+    print('Verifying src/azure-cli/service_name.json')
+    service_name_map = {}
+    for service_name in service_names:
+        command = service_name['Command']
+        service = service_name['AzureServiceName']
+        if not command.startswith('az '):
+            raise Exception('{} does not start with az!'.format(command))
+        if not service:
+            raise Exception('AzureServiceName of {} is empty!'.format(command))
+        service_name_map[command[3:]] = service
+    print('service_name_map:')
+    print(service_name_map)
+
+    # Check existence in service_name.json
+    for high_command in high_command_set:
+        if high_command not in service_name_map:
+            raise Exception('No entry of {} in service_name.json. Please add one to the file.'.format(high_command))
+
+
+if __name__ == "__main__":
+    main()

scripts/live_test/CLITest.yml

@@ -68,6 +68,8 @@ jobs:
         Target: container
       cosmosdb:
         Target: cosmosdb
+      databoxedge:
+        Target: databoxedge
       deploymentmanager:
         Target: deploymentmanager
       dla:
@@ -142,6 +144,8 @@ jobs:
         Target: sqlvm
       storage:
         Target: storage
+      synapse:
+        Target: synapse
       util:
         Target: util
       vm:

scripts/live_test/HISTORY.rst

@@ -3,6 +3,16 @@
 Release History
 ===============
 
+0.12.0 (4/15/2021)
+++++++
+
+* Add job databoxedge and synapse
+
+0.11.0 (1/18/2021)
+++++++
+
+* Add a job to automatically detect modules that have regression.
+
 0.10.0 (12/30/2020)
 ++++++