to catch up upstream master om 11th Mar 2021

.github/CODEOWNERS

@@ -261,6 +261,10 @@
 # ServiceLabel: %Digital Twins %Service Attention
 /sdk/digitaltwins/                              @drwill-ms @timtay-microsoft @abhipsaMisra @vinagesh @azabbasi @bikamani @barustum @jamdavi
 
+# PRLabel: %IoT Models Repository
+# ServiceLabel: %IoT Models Repository %Service Attention
+/sdk/modelsrepository/                              @drwill-ms @timtay-microsoft @abhipsaMisra @vinagesh @azabbasi @bikamani @barustum @jamdavi @digimaun
+
 # PRLabel: %TimeSeriesInsights
 # ServiceLabel: %TimeSeriesInsights %Service Attention
 /sdk/timeseriesinsights/                        @drwill-ms @timtay-microsoft @abhipsaMisra @vinagesh @azabbasi @bikamani @barustum @jamdavi @yeskarthik @rasidhan @dmdenmsft

common/Perf/Azure.Test.Perf/PerfProgram.cs

@@ -68,19 +68,6 @@ private static async Task Run(Type testType, PerfOptions options)
                 Console.WriteLine("Application started.");
             }
 
-            Console.WriteLine("=== Versions ===");
-            Console.WriteLine($"Runtime: {Environment.Version}");
-            var azureAssemblies = testType.Assembly.GetReferencedAssemblies()
-                .Where(a => a.Name.StartsWith("Azure", StringComparison.OrdinalIgnoreCase) || a.Name.StartsWith("Microsoft.Azure", StringComparison.OrdinalIgnoreCase))
-                .Where(a => !a.Name.Equals("Azure.Test.Perf", StringComparison.OrdinalIgnoreCase))
-                .OrderBy(a => a.Name);
-            foreach (var a in azureAssemblies)
-            {
-                var informationalVersion = FileVersionInfo.GetVersionInfo(Assembly.Load(a).Location).ProductVersion;
-                Console.WriteLine($"{a.Name}: {a.Version} ({informationalVersion})");
-            }
-            Console.WriteLine();
-
             Console.WriteLine("=== Options ===");
             Console.WriteLine(JsonSerializer.Serialize(options, options.GetType(), new JsonSerializerOptions()
             {
@@ -181,6 +168,46 @@ private static async Task Run(Type testType, PerfOptions options)
             {
                 cleanupStatusThread.Join();
             }
+
+            // I would prefer to print assembly versions at the start of testing, but they cannot be determined until
+            // code in each assembly has been executed, so this must wait until after testing is complete.
+            PrintAssemblyVersions(testType);
+        }
+
+        private static void PrintAssemblyVersions(Type testType)
+        {
+            Console.WriteLine("=== Versions ===");
+
+            Console.WriteLine($"Runtime:         {Environment.Version}");
+
+            var referencedAssemblies = testType.Assembly.GetReferencedAssemblies();
+
+            var azureLoadedAssemblies = AppDomain.CurrentDomain.GetAssemblies()
+                // Include all Track1 and Track2 assemblies
+                .Where(a => a.GetName().Name.StartsWith("Azure", StringComparison.OrdinalIgnoreCase) ||
+                            a.GetName().Name.StartsWith("Microsoft.Azure", StringComparison.OrdinalIgnoreCase))
+                // Exclude this perf framework
+                .Where(a => a != Assembly.GetExecutingAssembly())
+                // Exclude assembly containing the perf test itself
+                .Where(a => a != testType.Assembly)
+                // Exclude Azure.Core.TestFramework since it is only used to setup environment and should not impact results
+                .Where(a => !a.GetName().Name.Equals("Azure.Core.TestFramework", StringComparison.OrdinalIgnoreCase))
+                .OrderBy(a => a.GetName().Name);
+
+            foreach (var a in azureLoadedAssemblies)
+            {
+                var name = a.GetName().Name;
+                var referencedVersion = referencedAssemblies.Where(r => r.Name == name).SingleOrDefault()?.Version;
+                var loadedVersion = a.GetName().Version;
+                var informationalVersion = FileVersionInfo.GetVersionInfo(a.Location).ProductVersion;
+
+                Console.WriteLine($"{name}:");
+                Console.WriteLine($"  Referenced:    {referencedVersion}");
+                Console.WriteLine($"  Loaded:        {loadedVersion}");
+                Console.WriteLine($"  Informational: {informationalVersion}");
+            }
+
+            Console.WriteLine();
         }
 
         private static async Task RunTestsAsync(IPerfTest[] tests, PerfOptions options, string title, bool warmup = false)

common/SmokeTests/SmokeTest/SmokeTest.csproj

@@ -11,18 +11,21 @@
   </ItemGroup>
   <ItemGroup>
     <!-- Add an OverrideDailyVersion attribute to prevent the Update-Dependencies script from overwriting it with a daily build version -->
-    <PackageReference Include="Azure.Core" Version="1.9.0" />
-    <PackageReference Include="Azure.Identity" Version="1.4.0-beta.1" />
-    <PackageReference Include="Azure.Messaging.EventHubs" Version="5.3.0" />
-    <PackageReference Include="Azure.Messaging.EventHubs.Processor" Version="5.3.0" />
-    <PackageReference Include="Azure.Security.Keyvault.Secrets" Version="4.2.0-beta.3" />
-    <PackageReference Include="Azure.Storage.Blobs" Version="12.8.0" />
-    <PackageReference Include="Microsoft.Azure.Amqp" Version="2.4.9" />
+    <PackageReference Include="Azure.Core" Version="1.10.0" />
+    <PackageReference Include="Azure.Identity" Version="1.4.0-beta.4" />
+    <PackageReference Include="Azure.Messaging.EventHubs" Version="5.3.1" />
+    <PackageReference Include="Azure.Messaging.EventHubs.Processor" Version="5.3.1" />
+    <PackageReference Include="Azure.Security.Keyvault.Secrets" Version="4.2.0-beta.4" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.9.0-beta.1" />
+    <PackageReference Include="Microsoft.Azure.Amqp" Version="2.4.11" />
     <PackageReference Include="Microsoft.Azure.Devices" Version="1.20.1" />
     <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.10.0" />
+
+    <!-- This is needed to resolve a build conflict and force the correct version -->
+    <PackageReference Include="System.Net.NameResolution" Version="4.3.0" />
   </ItemGroup>
   <!-- Sample: IoT Hub Connection String Translation -->
   <ItemGroup>
     <Compile Include="$(MSBuildThisFileDirectory)..\..\..\samples\iothub-connect-to-eventhubs\**\*.cs" Link="Samples\IoTHubConnection\SharedSource\%(Filename)%(Extension)" />
   </ItemGroup>
-</Project>
+</Project>

common/SmokeTests/smoke-tests.yml

@@ -5,3 +5,4 @@ jobs:
   - template: /eng/pipelines/templates/jobs/smoke-tests.yml
     parameters:
       Daily: true
+      TimeoutInMinutes: 90

eng/.docsettings.yml

@@ -10,6 +10,7 @@ omitted_paths:
   - common/Perf/*
   - common/SmokeTests/*
   - common/Stress/*
+  - sdk/*/*/perf/*
   - sdk/*/samples/*
   - sdk/*/*/samples/*
   - sdk/*/swagger/*
@@ -34,7 +35,6 @@ known_presence_issues:
   - ['sdk/batch/Microsoft.Azure.Batch/Tools','#5499']
   - ['sdk/batch/Microsoft.Azure.Batch/Tools','#5499']
   - ['sdk/cognitiveservices/AnomalyDetector','#5499']
-  - ['sdk/cognitiveservices/FormRecognizer','#5499']
   - ['sdk/cognitiveservices/Knowledge.QnAMaker','#5499']
   - ['sdk/cognitiveservices/Language.LUIS.Authoring','#5499']
   - ['sdk/cognitiveservices/Language.LUIS.Runtime','#5499']
@@ -138,6 +138,7 @@ known_content_issues:
   - ['sdk/appconfiguration/README.md','azure-sdk-tools/issues/42']
   - ['sdk/appconfiguration/Azure.Data.AppConfiguration/tests/Readme.md','#5499']
   - ['sdk/storage/README.md','azure-sdk-tools/issues/42']
+  - ['sdk/cognitiveservices/FormRecognizer/README.md','#5499']
   - ['sdk/cognitiveservices/Language.TextAnalytics/src/README.md','#5499']
   - ['sdk/cognitiveservices/Personalizer/src/README.md','#5499']
   - ['sdk/batch/Microsoft.Azure.Batch/README.md','#5499']
@@ -151,6 +152,7 @@ known_content_issues:
   - ['sdk/eventgrid/Microsoft.Azure.Messaging.EventGrid.CloudNativeCloudEvents/README.md', '#15423']
   - ['sdk/eventhub/README.md','azure-sdk-tools/issues/42']
   - ['sdk/search/README.md','azure-sdk-tools/issues/42']
+  - ['sdk/search/Azure.Search.Documents/README.md','Credscan cleanup']
   - ['sdk/formrecognizer/Azure.AI.FormRecognizer/README.md','#5499']
   - ['sdk/formrecognizer/Azure.AI.FormRecognizer/README.md','#11492']
   - ['sdk/storage/Azure.Storage.Blobs/README.md','#11492']

eng/Packages.Data.props

@@ -11,7 +11,7 @@
     Dependency versions for Track 1 libraries.
   -->
   <ItemGroup Condition="'$(IsClientLibrary)' != 'true'">
-    <PackageReference Update="Microsoft.Azure.Amqp" Version="2.4.9" />
+    <PackageReference Update="Microsoft.Azure.Amqp" Version="2.4.11" />
     <PackageReference Update="Microsoft.Azure.Batch" Version="11.0.0" />
     <PackageReference Update="Microsoft.Azure.Devices.Client" Version="1.23.2" />
     <PackageReference Update="Microsoft.Azure.Devices" Version="1.19.0" />
@@ -54,7 +54,7 @@
     Dependency versions for Track 2, Azure.* libraries.
     Only packages that are approved dependencies should go here.
   -->
-  
+
   <ItemGroup Condition="'$(IsClientLibrary)' == 'true'">
 
     <!-- BCL packages -->
@@ -73,17 +73,17 @@
     <PackageReference Update="Microsoft.CSharp" Version="4.5.0" />
 
     <!-- Azure SDK packages -->
-    <PackageReference Update="Azure.Core" Version="1.9.0" />
+    <PackageReference Update="Azure.Core" Version="1.10.0" />
     <PackageReference Update="Azure.Core.Amqp" Version="1.0.0" />
-    <PackageReference Update="Azure.Core.Experimental" Version="0.1.0-preview.9" />
-    <PackageReference Update="Azure.Messaging.EventHubs" Version="5.3.0" />
+    <PackageReference Update="Azure.Core.Experimental" Version="0.1.0-preview.10" />
+    <PackageReference Update="Azure.Messaging.EventHubs" Version="5.3.1" />
     <PackageReference Update="Azure.Identity" Version="1.2.1" />
     <PackageReference Update="Azure.Security.KeyVault.Secrets" Version="4.0.2" />
     <PackageReference Update="Azure.Security.KeyVault.Keys" Version="4.0.2" />
     <PackageReference Update="Azure.Storage.Blobs" Version="12.8.0" />
-    
+
     <!-- Other approved packages -->
-    <PackageReference Update="Microsoft.Azure.Amqp" Version="2.4.9" />
+    <PackageReference Update="Microsoft.Azure.Amqp" Version="2.4.11" />
     <PackageReference Update="Microsoft.Identity.Client" Version="4.23.0" />
     <PackageReference Update="Microsoft.Identity.Client.Extensions.Msal" Version="2.16.6" />
     <!-- TODO: Make sure this package is arch-board approved -->
@@ -122,7 +122,7 @@
     All should have PrivateAssets="All" set so they don't become pacakge dependencies
   -->
   <ItemGroup>
-    <PackageReference Update="Microsoft.Azure.AutoRest.CSharp" Version="3.0.0-beta.20210218.1" PrivateAssets="All" />
+    <PackageReference Update="Microsoft.Azure.AutoRest.CSharp" Version="3.0.0-beta.20210311.1" PrivateAssets="All" />
     <PackageReference Update="Azure.ClientSdk.Analyzers" Version="0.1.1-dev.20200929.2" PrivateAssets="All" />
     <PackageReference Update="coverlet.collector" Version="1.3.0" PrivateAssets="All" />
     <PackageReference Update="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="3.3.1" PrivateAssets="All" />
@@ -141,18 +141,21 @@
     <PackageReference Update="ApprovalTests" Version="3.0.22" />
     <PackageReference Update="ApprovalUtilities" Version="3.0.22" />
     <PackageReference Update="Azure.Messaging.EventHubs" Version="5.2.0" />
+    <PackageReference Update="Azure.Messaging.EventGrid" Version="4.0.0-beta.5" />
     <PackageReference Update="Azure.ResourceManager.Compute" Version="1.0.0-preview.2" />
     <PackageReference Update="Azure.ResourceManager.Network" Version="1.0.0-preview.2" />
     <PackageReference Update="Azure.ResourceManager.Resources" Version="1.0.0-preview.2" />
     <PackageReference Update="Azure.ResourceManager.Storage" Version="1.0.0-preview.2" />
     <PackageReference Update="Azure.Search.Documents" Version="11.2.0" />
+    <PackageReference Update="Azure.Security.KeyVault.Secrets" Version="4.2.0-beta.4" />
     <PackageReference Update="Azure.Storage.Blobs" Version="12.6.0" />
     <PackageReference Update="Azure.Storage.Files.DataLake" Version="12.5.0" />
     <PackageReference Update="BenchmarkDotNet" Version="0.11.5" />
     <PackageReference Update="Castle.Core" Version="4.4.0" />
     <PackageReference Update="CommandLineParser" Version="2.8.0" />
     <PackageReference Update="FluentAssertions" Version="5.10.3" />
     <PackageReference Update="FsCheck.Xunit" Version="2.14.0" />
+    <PackageReference Update="Microsoft.Azure.ApplicationInsights.Query" Version="1.0.0" />
     <PackageReference Update="Microsoft.AspNetCore.Server.Kestrel" Version="2.2.0" />
     <PackageReference Update="Microsoft.AspNetCore.Server.WebListener" Version="1.0.2" />
     <PackageReference Update="Microsoft.Azure.Core.Spatial" Version="1.0.0" />
@@ -194,8 +197,8 @@
     <PackageReference Update="MSTest.TestFramework" Version="1.3.2" />
     <PackageReference Update="Newtonsoft.Json" Version="10.0.3" />
     <PackageReference Update="NSubstitute" Version="3.1.0" />
-    <PackageReference Update="NUnit" Version="3.12.0" />
-    <PackageReference Update="NUnit3TestAdapter" Version="3.13.0" />
+    <PackageReference Update="NUnit" Version="3.13.1" />
+    <PackageReference Update="NUnit3TestAdapter" Version="3.17.0" />
     <PackageReference Update="OpenTelemetry.Extensions.Hosting" Version="1.0.0-rc2" />
     <PackageReference Update="OpenTelemetry.Instrumentation.AspNetCore" Version="1.0.0-rc2" />
     <PackageReference Update="OpenTelemetry.Instrumentation.Http" Version="1.0.0-rc2" />

eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml

@@ -12,16 +12,40 @@ parameters:
   default: []
 - name: JobTemplatePath
   type: string
+# Set this to false to do a full checkout for private repositories with the azure pipelines service connection
+- name: SparseCheckout
+  type: boolean
+  default: true
+- name: SparseCheckoutPaths
+  type: object
+  default: []
+- name: Pool
+  type: string
+  default: azsdk-pool-mms-ubuntu-1804-general
+- name: OsVmImage
+  type: string
+  default: MMSUbuntu18.04
 
 jobs:
 - job: generate_matrix
   variables:
     displayNameFilter: $[ coalesce(variables.jobMatrixFilter, '.*') ]
   pool:
-    name: Azure Pipelines
-    vmImage: ubuntu-18.04
+    name: ${{ parameters.Pool }}
+    vmImage: ${{ parameters.OsVmImage }}
   displayName: Generate Job Matrix
   steps:
+    # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
+    # as we require the github service connection to be loaded.
+    - ${{ if and(parameters.SparseCheckout, not(contains(variables['Build.DefinitionName'], '-pr - '))) }}:
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+        parameters:
+          ${{ if ne(length(parameters.SparseCheckoutPaths), 0) }}:
+            Paths: ${{ parameters.SparseCheckoutPaths }}
+          ${{ if and(eq(length(parameters.SparseCheckoutPaths), 0), ne(parameters.AdditionalParameters.ServiceDirectory, '')) }}:
+            Paths:
+              - ${{ parameters.AdditionalParameters.ServiceDirectory }}
+
     - ${{ each config in parameters.MatrixConfigs }}:
       - ${{ if eq(config.GenerateVMJobs, 'true') }}:
         - task: Powershell@2

eng/common/pipelines/templates/steps/check-spelling.yml

@@ -0,0 +1,27 @@
+# Checks spelling of files that changed between the current state of the repo
+# and some ref (branch, tag, etc.) or commit hash. Only runs on PRs.
+# ContinueOnError   - true: Pipeline warns on spelling error 
+#                     false: Pipeline fails on spelling error
+# TargetBranch      - Target ref (e.g. master) to compare to create file change
+#                     list.
+# CspellConfigPath  - Path to cspell.json config location
+
+parameters:
+  ContinueOnError: true
+  TargetBranch: $(System.PullRequest.TargetBranch)
+  SourceBranch: HEAD
+  CspellConfigPath: ./.vscode/cspell.json
+
+steps:
+  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+    - task: PowerShell@2
+      displayName: Check spelling (cspell)
+      continueOnError: ${{ parameters.ContinueOnError }}
+      inputs: 
+        targetType: filePath
+        filePath: eng/common/scripts/check-spelling-in-changed-files.ps1
+        arguments: >- 
+          -TargetBranch "origin/$("${{ parameters.TargetBranch }}" -replace "refs/heads/")"
+          -SourceBranch ${{ parameters.SourceBranch }}
+          -CspellConfigPath ${{ parameters.CspellConfigPath }}
+        pwsh: true

eng/common/pipelines/templates/steps/create-apireview.yml

@@ -19,4 +19,3 @@ steps:
         workingDirectory: $(Pipeline.Workspace)
       displayName: Create API Review for ${{ artifact.name}}
       condition: and(succeededOrFailed(), ne(variables['Skip.CreateApiReview'], 'true') , ne(variables['Build.Reason'],'PullRequest'), eq(variables['System.TeamProject'], 'internal'))
-      continueOnError: true

eng/common/pipelines/templates/steps/install-pipeline-generation.yml

@@ -9,7 +9,7 @@ steps:
   - script: >
       dotnet tool install
       Azure.Sdk.Tools.PipelineGenerator
-      --version 1.0.2-dev.20210219.1
+      --version 1.0.2-dev.20210309.1
       --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk/nuget/v3/index.json
       --tool-path ${{parameters.ToolPath}}
     workingDirectory: $(Pipeline.Workspace)/pipeline-generator

eng/common/pipelines/templates/steps/set-test-pipeline-version.yml

@@ -0,0 +1,15 @@
+parameters:
+  PackageName: ''
+  ServiceDirectory: ''
+  TestPipeline: false
+
+steps:
+- ${{if eq(parameters.TestPipeline, 'true')}}:
+    - task: PowerShell@2
+      displayName: Prep template pipeline for release
+      condition: succeeded()
+      inputs:
+        pwsh: true
+        workingDirectory: $(Build.SourcesDirectory)
+        filePath: $(Build.SourcesDirectory)/eng/common/scripts/SetTestPipelineVersion.ps1
+        arguments: '-BuildID $(Build.BuildId) -PackageName ${{ parameters.PackageName }} -ServiceDirectory ${{ parameters.ServiceDirectory }}'

eng/common/pipelines/templates/steps/sparse-checkout.yml

@@ -0,0 +1,34 @@
+parameters:
+  - name: Paths
+    type: object
+    default: []
+  - name: Repositories
+    type: object
+    default:
+      - Name: $(Build.Repository.Name)
+        Commitish: $(Build.SourceVersion)
+        WorkingDirectory: $(System.DefaultWorkingDirectory)
+
+steps:
+  - checkout: none
+
+  - ${{ each repo in parameters.Repositories }}:
+    - pwsh: |
+        $dir = "${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}"
+        New-Item $dir -ItemType Directory -Force
+
+    - pwsh: |
+        git clone --no-checkout --filter=tree:0 git://github.com/${{ repo.Name }} .
+        git sparse-checkout init
+        git sparse-checkout set eng
+      displayName: Init sparse checkout ${{ repo.Name }}
+      workingDirectory: ${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}
+
+    - ${{ each path in parameters.Paths }}:
+      - pwsh: git sparse-checkout add ${{ path }}
+        displayName: Add sparse checkout path ${{ path }}
+        workingDirectory: ${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}
+
+    - pwsh: git checkout ${{ repo.Commitish }}
+      displayName: Sparse checkout at ${{ repo.Commitish }}
+      workingDirectory: ${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}

eng/common/scripts/ChangeLog-Operations.ps1

@@ -193,15 +193,14 @@ function Set-ChangeLogContent {
 
   try
   {
-    $VersionsSorted = [AzureEngSemanticVersion]::SortVersionStrings($ChangeLogEntries.Keys)
+    $ChangeLogEntries = $ChangeLogEntries.Values | Sort-Object -Descending -Property ReleaseStatus, ReleaseVersion
   }
   catch {
     LogError "Problem sorting version in ChangeLogEntries"
     return
   }
 
-  foreach ($version in $VersionsSorted) {
-    $changeLogEntry = $ChangeLogEntries[$version]
+  foreach ($changeLogEntry in $ChangeLogEntries) {
     $changeLogContent += $changeLogEntry.ReleaseTitle
     if ($changeLogEntry.ReleaseContent.Count -eq 0) {
       $changeLogContent += @("","")