Internal Gates: Fixes Tests and update pipelines

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs

@@ -363,7 +363,7 @@ public async Task ReadManyMultiplePK()
  {
  IReadOnlyList<string> pkPaths = new List<string> { "/pk", "/description" };
  ContainerProperties containerSettings = new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPaths: pkPaths);
- Container container = await this.database.CreateContainerAsync(this.containerSettings);
+ Container container = await this.database.CreateContainerAsync(containerSettings);
 
  for (int i = 0; i < 5; i++)
  {

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/BenchmarkResults.json

@@ -19,7 +19,7 @@
  "MockedItemBenchmark.ReadFeed;[Type=Stream]": 42546.0,
  "MockedItemBenchmark.ReadItemExists;[Type=OfT]": 41814.0,
  "MockedItemBenchmark.ReadItemExists;[Type=OfTCustom]": 41806.0,
- "MockedItemBenchmark.ReadItemExists;[Type=OfTWithDiagnosticsToString]": 70752.0,
+ "MockedItemBenchmark.ReadItemExists;[Type=OfTWithDiagnosticsToString]": 74898.0,
  "MockedItemBenchmark.ReadItemExists;[Type=Stream]": 34464.0,
  "MockedItemBenchmark.ReadItemNotExists;[Type=OfT]": 52826.0,
  "MockedItemBenchmark.ReadItemNotExists;[Type=OfTCustom]": 52864.0,

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/PerformanceValidation.cs

@@ -72,40 +72,60 @@ public static int ValidateSummaryResultsAgainstBaseline(SortedDictionary<string,
 
  currentDirectory += PerformanceValidation.DirectoryPath;
 
- // Always write the updated version. This will change with each run.
- string currentBenchmarkResults = JsonConvert.SerializeObject(operationToMemoryAllocated, Formatting.Indented);
- File.WriteAllText(currentDirectory + PerformanceValidation.CurrentBenchmarkResultsFileName, currentBenchmarkResults);
-
  string baselineJson = File.ReadAllText(currentDirectory + PerformanceValidation.BaselineBenchmarkResultsFileName);
  Dictionary<string, double> baselineBenchmarkResults = JsonConvert.DeserializeObject<Dictionary<string, double>>(baselineJson);
 
- if (baselineBenchmarkResults.Count != operationToMemoryAllocated.Count)
- {
- Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
- return 1;
- }
-
- foreach(KeyValuePair<string, double> currentResult in operationToMemoryAllocated)
+ List<string> failures = new List<string>();
+ SortedDictionary<string, double> updatedBaseline = new SortedDictionary<string, double>();
+ foreach (KeyValuePair<string, double> currentResult in operationToMemoryAllocated)
  {
- double baselineResult = baselineBenchmarkResults[currentResult.Key];
+ if(!baselineBenchmarkResults.TryGetValue(
+ currentResult.Key, 
+ out double baselineResult))
+ {
+ updatedBaseline.Add(currentResult.Key, currentResult.Value);
+ continue;
+ }
 
  // Add 5% buffer to avoid minor variation between test runs
- double diff = currentResult.Value - baselineResult;
+ double diff = Math.Abs(currentResult.Value - baselineResult);
  double maxAllowedDiff = baselineResult * .05;
+ double minDiffToUpdatebaseLine = baselineResult * .02;
  if (diff > maxAllowedDiff)
  {
- Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
- return 1;
+ updatedBaseline.Add(currentResult.Key, currentResult.Value);
+ failures.Add($"{currentResult.Key}: {currentResult.Value}");
  }
- else if (-diff > maxAllowedDiff)
+ else if(diff > minDiffToUpdatebaseLine)
  {
- Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
- return 1;
+ // Update the value if it is greater than 2% difference.
+ // This reduces the noise and make it easier to see which values actually changed
+ updatedBaseline.Add(currentResult.Key, currentResult.Value);
+ }
+ else
+ {
+ // Use the baseline if the value didn't change by more than 2% to avoid updating values unnecessarily
+ // This makes it easier to see which values actually need to be updated.
+ updatedBaseline.Add(currentResult.Key, baselineResult);
  }
  }
 
+ // Always write the updated version. This will change with each run.
+ string currentBenchmarkResults = JsonConvert.SerializeObject(updatedBaseline, Formatting.Indented);
+ File.WriteAllText(currentDirectory + PerformanceValidation.CurrentBenchmarkResultsFileName, currentBenchmarkResults);
  Console.WriteLine("Current benchmark results: " + currentBenchmarkResults);
 
+ if (failures.Any())
+ {
+ Console.WriteLine(PerformanceValidation.UpdateMessage);
+ foreach(string failure in failures)
+ {
+ Console.WriteLine(failure);
+ }
+
+ return 1;
+ }
+
  return 0;
  }
  }

azure-pipelines-functional.yml

@@ -14,7 +14,7 @@ pr:
 variables:
  DebugArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal '
  ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal '
- VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
+ VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
 
 
 jobs: 

azure-pipelines-nightly.yml

@@ -14,7 +14,7 @@ schedules:
 
 variables:
  ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal ' 
- VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
+ VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
  BuildConfiguration: Release 
  IsNightly: true 
 

azure-pipelines.yml

@@ -13,7 +13,7 @@ pr:
 variables:
  DebugArguments: ' --filter "TestCategory!=Quarantine & TestCategory!=Functional" --verbosity normal '
  ReleaseArguments: ' --filter "TestCategory!=Quarantine & TestCategory!=Functional" --verbosity normal '
- VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
+ VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
 
 
 jobs: