Internal Gates: Fixes Tests and update pipelines

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs

@@ -363,7 +363,7 @@ public async Task ReadManyMultiplePK()
         {
             IReadOnlyList<string> pkPaths = new List<string> { "/pk", "/description" };
             ContainerProperties containerSettings = new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPaths: pkPaths);
-            Container container = await this.database.CreateContainerAsync(this.containerSettings);
+            Container container = await this.database.CreateContainerAsync(containerSettings);
 
             for (int i = 0; i < 5; i++)
             {

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/PerformanceValidation.cs

@@ -72,40 +72,60 @@ public static int ValidateSummaryResultsAgainstBaseline(SortedDictionary<string,
 
             currentDirectory += PerformanceValidation.DirectoryPath;
 
-            // Always write the updated version. This will change with each run.
-            string currentBenchmarkResults = JsonConvert.SerializeObject(operationToMemoryAllocated, Formatting.Indented);
-            File.WriteAllText(currentDirectory + PerformanceValidation.CurrentBenchmarkResultsFileName, currentBenchmarkResults);
-
             string baselineJson = File.ReadAllText(currentDirectory + PerformanceValidation.BaselineBenchmarkResultsFileName);
             Dictionary<string, double> baselineBenchmarkResults = JsonConvert.DeserializeObject<Dictionary<string, double>>(baselineJson);
 
-            if (baselineBenchmarkResults.Count != operationToMemoryAllocated.Count)
-            {
-                Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
-                return 1;
-            }
-
-            foreach(KeyValuePair<string, double> currentResult in operationToMemoryAllocated)
+            List<string> failures = new List<string>();
+            SortedDictionary<string, double> updatedBaseline = new SortedDictionary<string, double>();
+            foreach (KeyValuePair<string, double> currentResult in operationToMemoryAllocated)
             {
-                double baselineResult = baselineBenchmarkResults[currentResult.Key];
+                if(!baselineBenchmarkResults.TryGetValue(
+                    currentResult.Key, 
+                    out double baselineResult))
+                {
+                    updatedBaseline.Add(currentResult.Key, currentResult.Value);
+                    continue;
+                }
 
                 // Add 5% buffer to avoid minor variation between test runs
-                double diff = currentResult.Value - baselineResult;
+                double diff = Math.Abs(currentResult.Value - baselineResult);
                 double maxAllowedDiff = baselineResult * .05;
+                double minDiffToUpdatebaseLine = baselineResult * .02;
                 if (diff > maxAllowedDiff)
                 {
-                    Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
-                    return 1;
+                    updatedBaseline.Add(currentResult.Key, currentResult.Value);
+                    failures.Add($"{currentResult.Key}: {currentResult.Value}");
                 }
-                else if (-diff > maxAllowedDiff)
+                else if(diff > minDiffToUpdatebaseLine)
                 {
-                    Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
-                    return 1;
+                    // Update the value if it is greater than 2% difference.
+                    // This reduces the noise and make it easier to see which values actually changed
+                    updatedBaseline.Add(currentResult.Key, currentResult.Value);
+                }
+                else
+                {
+                    // Use the baseline if the value didn't change by more than 2% to avoid updating values unnecessarily
+                    // This makes it easier to see which values actually need to be updated.
+                    updatedBaseline.Add(currentResult.Key, baselineResult);
                 }
             }
 
+            // Always write the updated version. This will change with each run.
+            string currentBenchmarkResults = JsonConvert.SerializeObject(updatedBaseline, Formatting.Indented);
+            File.WriteAllText(currentDirectory + PerformanceValidation.CurrentBenchmarkResultsFileName, currentBenchmarkResults);
             Console.WriteLine("Current benchmark results: " + currentBenchmarkResults);
 
+            if (failures.Any())
+            {
+                Console.WriteLine(PerformanceValidation.UpdateMessage);
+                foreach(string failure in failures)
+                {
+                    Console.WriteLine(failure);
+                }
+
+                return 1;
+            }
+
             return 0;
         }
     }

azure-pipelines-functional.yml

@@ -14,7 +14,7 @@ pr:
 variables:
   DebugArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal '
   ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal '
-  VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
+  VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
 
 
 jobs: 

azure-pipelines-nightly.yml

@@ -14,7 +14,7 @@ schedules:
 
 variables:
   ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal ' 
-  VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
+  VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
   BuildConfiguration: Release 
   IsNightly: true 
 

azure-pipelines.yml

@@ -13,7 +13,7 @@ pr:
 variables:
   DebugArguments: ' --filter "TestCategory!=Quarantine & TestCategory!=Functional" --verbosity normal '
   ReleaseArguments: ' --filter "TestCategory!=Quarantine & TestCategory!=Functional" --verbosity normal '
-  VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
+  VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
 
 
 jobs: