diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs index 9e778d9e00..526cbfeebe 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs @@ -363,7 +363,7 @@ public async Task ReadManyMultiplePK() { IReadOnlyList pkPaths = new List { "/pk", "/description" }; ContainerProperties containerSettings = new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPaths: pkPaths); - Container container = await this.database.CreateContainerAsync(this.containerSettings); + Container container = await this.database.CreateContainerAsync(containerSettings); for (int i = 0; i < 5; i++) { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/BenchmarkResults.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/BenchmarkResults.json index 594fccf00a..78374c2155 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/BenchmarkResults.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/BenchmarkResults.json @@ -19,7 +19,7 @@ "MockedItemBenchmark.ReadFeed;[Type=Stream]": 42546.0, "MockedItemBenchmark.ReadItemExists;[Type=OfT]": 41814.0, "MockedItemBenchmark.ReadItemExists;[Type=OfTCustom]": 41806.0, - "MockedItemBenchmark.ReadItemExists;[Type=OfTWithDiagnosticsToString]": 70752.0, + "MockedItemBenchmark.ReadItemExists;[Type=OfTWithDiagnosticsToString]": 74898.0, "MockedItemBenchmark.ReadItemExists;[Type=Stream]": 34464.0, "MockedItemBenchmark.ReadItemNotExists;[Type=OfT]": 52826.0, "MockedItemBenchmark.ReadItemNotExists;[Type=OfTCustom]": 52864.0, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/PerformanceValidation.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/PerformanceValidation.cs index 75407e9f6b..131c7ca251 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/PerformanceValidation.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/PerformanceValidation.cs @@ -72,40 +72,60 @@ public static int ValidateSummaryResultsAgainstBaseline(SortedDictionary baselineBenchmarkResults = JsonConvert.DeserializeObject>(baselineJson); - if (baselineBenchmarkResults.Count != operationToMemoryAllocated.Count) - { - Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults); - return 1; - } - - foreach(KeyValuePair currentResult in operationToMemoryAllocated) + List failures = new List(); + SortedDictionary updatedBaseline = new SortedDictionary(); + foreach (KeyValuePair currentResult in operationToMemoryAllocated) { - double baselineResult = baselineBenchmarkResults[currentResult.Key]; + if(!baselineBenchmarkResults.TryGetValue( + currentResult.Key, + out double baselineResult)) + { + updatedBaseline.Add(currentResult.Key, currentResult.Value); + continue; + } // Add 5% buffer to avoid minor variation between test runs - double diff = currentResult.Value - baselineResult; + double diff = Math.Abs(currentResult.Value - baselineResult); double maxAllowedDiff = baselineResult * .05; + double minDiffToUpdatebaseLine = baselineResult * .02; if (diff > maxAllowedDiff) { - Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults); - return 1; + updatedBaseline.Add(currentResult.Key, currentResult.Value); + failures.Add($"{currentResult.Key}: {currentResult.Value}"); } - else if (-diff > maxAllowedDiff) + else if(diff > minDiffToUpdatebaseLine) { - Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults); - return 1; + // Update the value if it is greater than 2% difference. + // This reduces the noise and make it easier to see which values actually changed + updatedBaseline.Add(currentResult.Key, currentResult.Value); + } + else + { + // Use the baseline if the value didn't change by more than 2% to avoid updating values unnecessarily + // This makes it easier to see which values actually need to be updated. + updatedBaseline.Add(currentResult.Key, baselineResult); } } + // Always write the updated version. This will change with each run. + string currentBenchmarkResults = JsonConvert.SerializeObject(updatedBaseline, Formatting.Indented); + File.WriteAllText(currentDirectory + PerformanceValidation.CurrentBenchmarkResultsFileName, currentBenchmarkResults); Console.WriteLine("Current benchmark results: " + currentBenchmarkResults); + if (failures.Any()) + { + Console.WriteLine(PerformanceValidation.UpdateMessage); + foreach(string failure in failures) + { + Console.WriteLine(failure); + } + + return 1; + } + return 0; } } diff --git a/azure-pipelines-functional.yml b/azure-pipelines-functional.yml index 42de771703..355db29ba7 100644 --- a/azure-pipelines-functional.yml +++ b/azure-pipelines-functional.yml @@ -14,7 +14,7 @@ pr: variables: DebugArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal ' ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal ' - VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops + VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops jobs: diff --git a/azure-pipelines-nightly.yml b/azure-pipelines-nightly.yml index 10de80b127..8db3ff01e6 100644 --- a/azure-pipelines-nightly.yml +++ b/azure-pipelines-nightly.yml @@ -14,7 +14,7 @@ schedules: variables: ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal ' - VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops + VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops BuildConfiguration: Release IsNightly: true diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3266d51894..a06b5c2a18 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,7 @@ pr: variables: DebugArguments: ' --filter "TestCategory!=Quarantine & TestCategory!=Functional" --verbosity normal ' ReleaseArguments: ' --filter "TestCategory!=Quarantine & TestCategory!=Functional" --verbosity normal ' - VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops + VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops jobs: