From 9de35cf0b1a178f789cf557e97b116e622a4ef8d Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 5 Aug 2022 16:43:55 -0700 Subject: [PATCH 01/17] Add sparse checkout to perf template --- eng/common/pipelines/templates/jobs/perf2.yml | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 eng/common/pipelines/templates/jobs/perf2.yml diff --git a/eng/common/pipelines/templates/jobs/perf2.yml b/eng/common/pipelines/templates/jobs/perf2.yml new file mode 100644 index 0000000000..c62e37f57c --- /dev/null +++ b/eng/common/pipelines/templates/jobs/perf2.yml @@ -0,0 +1,147 @@ +parameters: +- name: RepoName + type: string + default: '' +- name: Variables + type: object + default: [] +- name: OperatingSystems + type: string + default: 'Linux' +- name: Language + type: string + default: '' +- name: InstallLanguageSteps + type: stepList + default: [] +- name: ServiceDirectory + type: string + default: '' +- name: Services + type: string + default: '' +- name: PackageVersions + type: string + default: '.*' +- name: Tests + type: string + default: '.*' +- name: Arguments + type: string + default: '.*' +- name: Iterations + type: number + default: '5' +- name: AdditionalArguments + type: string + default: '' +- name: EnvVars + type: object + default: {} + +jobs: +- job: Perf + timeoutInMinutes: 360 + strategy: + matrix: + ${{ if contains(parameters.OperatingSystems, 'Linux') }}: + Linux: + Pool: 'azsdk-pool-mms-ubuntu-2004-perf' + OsVmImage: 'MMSUbuntu20.04' + MatrixName: 'Linux' + ${{ if contains(parameters.OperatingSystems, 'Windows') }}: + Windows: + Pool: 'azsdk-pool-mms-win-2019-perf' + OsVmImage: 'MMS2019' + MatrixName: 'Windows' + variables: + - ${{ parameters.Variables }} + pool: + name: $(Pool) + vmImage: $(OSVmImage) + steps: + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - '/*' + - '!sdk/**/test-recordings/*' + - '!sdk/**/recordings/*' + - '!sdk/**/SessionRecords/*' + - '!sdk/**/session-records/*' + Repositories: + - Name: ${{ parameters.RepoName }} + Commitish: main + WorkingDirectory: $(System.DefaultWorkingDirectory) + - Name: Azure/azure-sdk-tools + Commitish: $(Build.SourceVersion) + WorkingDirectory: $(System.DefaultWorkingDirectory)/azure-sdk-tools + + - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml + parameters: + AgentImage: $(OSVmImage) + + - ${{ parameters.InstallLanguageSteps }} + + - template: /eng/common/TestResources/deploy-test-resources.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Location: westus + ResourceType: perf + + - pwsh: | + set-content -path config.yml -value "WorkingDirectories:" + add-content -path config.yml -value " ${{ parameters.Language }}: $(Agent.BuildDirectory)/s" + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation + displayName: Create config.yml + + - script: >- + dotnet run -- run + --no-sync + --languages ${{ parameters.Language }} + --services "${{ parameters.Services }}" + --package-versions "${{ parameters.PackageVersions }}" + --tests "${{ parameters.Tests }}" + --arguments "${{ parameters.Arguments }}" + --iterations ${{ parameters.Iterations }} + ${{ parameters.AdditionalArguments }} + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation + env: + ${{ each var in parameters.EnvVars }}: + ${{ var.key }}: ${{ var.value }} + displayName: Run perf tests + + - pwsh: | + get-content results.txt + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results + displayName: Print results.txt + condition: always() + + - pwsh: | + get-content results.csv + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results + displayName: Print results.csv + condition: always() + + - pwsh: | + get-content results.md + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results + displayName: Print results.md + condition: always() + + - pwsh: | + get-content results.json + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results + displayName: Print results.json + condition: always() + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results + artifactName: results-$(MatrixName) + condition: always() + + - template: /eng/common/TestResources/remove-test-resources.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + ResourceType: perf From ef1c7c231c7fd1f5f334f5c0c5d8887507b741f0 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 5 Aug 2022 17:12:03 -0700 Subject: [PATCH 02/17] Rename base template, add parameters --- .../jobs/{perf2.yml => perf-base.yml} | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) rename eng/common/pipelines/templates/jobs/{perf2.yml => perf-base.yml} (91%) diff --git a/eng/common/pipelines/templates/jobs/perf2.yml b/eng/common/pipelines/templates/jobs/perf-base.yml similarity index 91% rename from eng/common/pipelines/templates/jobs/perf2.yml rename to eng/common/pipelines/templates/jobs/perf-base.yml index c62e37f57c..0737b19b89 100644 --- a/eng/common/pipelines/templates/jobs/perf2.yml +++ b/eng/common/pipelines/templates/jobs/perf-base.yml @@ -1,5 +1,14 @@ parameters: -- name: RepoName +- name: JobName + type: string + default: 'Perf' +- name: LanguageRepoName + type: string + default: '' +- name: LanguageRepoCommitish + type: string + default: '' +- name: ToolsRepoCommitish type: string default: '' - name: Variables @@ -40,7 +49,7 @@ parameters: default: {} jobs: -- job: Perf +- job: ${{ parameters.JobName }} timeoutInMinutes: 360 strategy: matrix: @@ -70,11 +79,11 @@ jobs: - '!sdk/**/SessionRecords/*' - '!sdk/**/session-records/*' Repositories: - - Name: ${{ parameters.RepoName }} - Commitish: main + - Name: ${{ parameters.LanguageRepoName }} + Commitish: ${{ parameters.LanguageRepoCommitish }} WorkingDirectory: $(System.DefaultWorkingDirectory) - Name: Azure/azure-sdk-tools - Commitish: $(Build.SourceVersion) + Commitish: ${{ parameters.LanguageRepoCommitish }} WorkingDirectory: $(System.DefaultWorkingDirectory)/azure-sdk-tools - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml From e2a4b95cb41621d8ab214bc1f4eb6bc4a8c30fdf Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 5 Aug 2022 17:13:44 -0700 Subject: [PATCH 03/17] Add comment --- eng/common/pipelines/templates/jobs/perf.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/common/pipelines/templates/jobs/perf.yml b/eng/common/pipelines/templates/jobs/perf.yml index 3d3a3f6e31..3589527d5f 100644 --- a/eng/common/pipelines/templates/jobs/perf.yml +++ b/eng/common/pipelines/templates/jobs/perf.yml @@ -1,3 +1,5 @@ +# TODO: Extend perf-base.yml + parameters: - name: Variables type: object From ffdf626070fcf8ee5d3f6958f830740a8e99b183 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 5 Aug 2022 17:25:51 -0700 Subject: [PATCH 04/17] Add TODO --- eng/common/pipelines/templates/jobs/perf-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/common/pipelines/templates/jobs/perf-base.yml b/eng/common/pipelines/templates/jobs/perf-base.yml index 0737b19b89..d8a12f1ae7 100644 --- a/eng/common/pipelines/templates/jobs/perf-base.yml +++ b/eng/common/pipelines/templates/jobs/perf-base.yml @@ -144,6 +144,7 @@ jobs: displayName: Print results.json condition: always() + # TODO: Add $(Language) to disambiguate - task: PublishPipelineArtifact@1 inputs: targetPath: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results From 105660627fdc7af53ed55438032fb03adab2a97b Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 5 Aug 2022 17:31:09 -0700 Subject: [PATCH 05/17] Fix copy/pase bug, add language to artifacts path --- eng/common/pipelines/templates/jobs/perf-base.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/perf-base.yml b/eng/common/pipelines/templates/jobs/perf-base.yml index d8a12f1ae7..ec0787655d 100644 --- a/eng/common/pipelines/templates/jobs/perf-base.yml +++ b/eng/common/pipelines/templates/jobs/perf-base.yml @@ -83,7 +83,7 @@ jobs: Commitish: ${{ parameters.LanguageRepoCommitish }} WorkingDirectory: $(System.DefaultWorkingDirectory) - Name: Azure/azure-sdk-tools - Commitish: ${{ parameters.LanguageRepoCommitish }} + Commitish: ${{ parameters.ToolsRepoCommitish }} WorkingDirectory: $(System.DefaultWorkingDirectory)/azure-sdk-tools - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml @@ -144,11 +144,10 @@ jobs: displayName: Print results.json condition: always() - # TODO: Add $(Language) to disambiguate - task: PublishPipelineArtifact@1 inputs: targetPath: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - artifactName: results-$(MatrixName) + artifactName: results-$(Language)-$(MatrixName) condition: always() - template: /eng/common/TestResources/remove-test-resources.yml From afa61961099b013f4ce76aa23d48d09a1bd1f2c1 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 5 Aug 2022 17:46:00 -0700 Subject: [PATCH 06/17] Fix language ref --- eng/common/pipelines/templates/jobs/perf-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/perf-base.yml b/eng/common/pipelines/templates/jobs/perf-base.yml index ec0787655d..c41411c9b8 100644 --- a/eng/common/pipelines/templates/jobs/perf-base.yml +++ b/eng/common/pipelines/templates/jobs/perf-base.yml @@ -147,7 +147,7 @@ jobs: - task: PublishPipelineArtifact@1 inputs: targetPath: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - artifactName: results-$(Language)-$(MatrixName) + artifactName: results-${{ parameters.Language }}-$(MatrixName) condition: always() - template: /eng/common/TestResources/remove-test-resources.yml From 1b4eb7ae32648b2cf248e625a9427e552056b905 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 5 Aug 2022 17:49:30 -0700 Subject: [PATCH 07/17] Fix python language --- eng/common/pipelines/templates/jobs/perf-base.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/perf-base.yml b/eng/common/pipelines/templates/jobs/perf-base.yml index c41411c9b8..c9348f0a24 100644 --- a/eng/common/pipelines/templates/jobs/perf-base.yml +++ b/eng/common/pipelines/templates/jobs/perf-base.yml @@ -2,6 +2,9 @@ parameters: - name: JobName type: string default: 'Perf' +- name: Language + type: string + default: '' - name: LanguageRepoName type: string default: '' @@ -17,9 +20,6 @@ parameters: - name: OperatingSystems type: string default: 'Linux' -- name: Language - type: string - default: '' - name: InstallLanguageSteps type: stepList default: [] From 3c6970641f74c78d436e94e005c8c2a6f8f0824d Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 5 Aug 2022 17:54:32 -0700 Subject: [PATCH 08/17] perf.yml extends perf-base.yml --- eng/common/pipelines/templates/jobs/perf.yml | 130 +++---------------- 1 file changed, 21 insertions(+), 109 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/perf.yml b/eng/common/pipelines/templates/jobs/perf.yml index 3589527d5f..505eae90a2 100644 --- a/eng/common/pipelines/templates/jobs/perf.yml +++ b/eng/common/pipelines/templates/jobs/perf.yml @@ -1,15 +1,13 @@ -# TODO: Extend perf-base.yml - parameters: +- name: Language + type: string + default: '' - name: Variables type: object default: [] - name: OperatingSystems type: string default: 'Linux' -- name: Language - type: string - default: '' - name: InstallLanguageSteps type: stepList default: [] @@ -38,107 +36,21 @@ parameters: type: object default: {} -resources: - repositories: - - repository: azure-sdk-tools - type: github - endpoint: Azure - name: Azure/azure-sdk-tools - ref: main - -variables: -- ${{ parameters.Variables }} - -jobs: -- job: Perf - timeoutInMinutes: 360 - strategy: - matrix: - ${{ if contains(parameters.OperatingSystems, 'Linux') }}: - Linux: - Pool: 'azsdk-pool-mms-ubuntu-2004-perf' - OsVmImage: 'MMSUbuntu20.04' - MatrixName: 'Linux' - ${{ if contains(parameters.OperatingSystems, 'Windows') }}: - Windows: - Pool: 'azsdk-pool-mms-win-2019-perf' - OsVmImage: 'MMS2019' - MatrixName: 'Windows' - pool: - name: $(Pool) - vmImage: $(OSVmImage) - steps: - - checkout: self - path: s - - - checkout: azure-sdk-tools - path: s/azure-sdk-tools - - - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml - parameters: - AgentImage: $(OSVmImage) - - - ${{ parameters.InstallLanguageSteps }} - - - template: /eng/common/TestResources/deploy-test-resources.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - Location: westus - ResourceType: perf - - - pwsh: | - set-content -path config.yml -value "WorkingDirectories:" - add-content -path config.yml -value " ${{ parameters.Language }}: $(Agent.BuildDirectory)/s" - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation - displayName: Create config.yml - - - script: >- - dotnet run -- run - --no-sync - --languages ${{ parameters.Language }} - --services "${{ parameters.Services }}" - --package-versions "${{ parameters.PackageVersions }}" - --tests "${{ parameters.Tests }}" - --arguments "${{ parameters.Arguments }}" - --iterations ${{ parameters.Iterations }} - ${{ parameters.AdditionalArguments }} - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation - env: - ${{ each var in parameters.EnvVars }}: - ${{ var.key }}: ${{ var.value }} - displayName: Run perf tests - - - pwsh: | - get-content results.txt - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - displayName: Print results.txt - condition: always() - - - pwsh: | - get-content results.csv - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - displayName: Print results.csv - condition: always() - - - pwsh: | - get-content results.md - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - displayName: Print results.md - condition: always() - - - pwsh: | - get-content results.json - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - displayName: Print results.json - condition: always() - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - artifactName: results-$(MatrixName) - condition: always() - - - template: /eng/common/TestResources/remove-test-resources.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - ResourceType: perf +extends: + template: perf-base.yml + parameters: + Language: ${{ parameters.Language }} + LanguageRepoName: $(Build.Repository.Name) + LanguageRepoCommitish: $(Build.SourceVersion) + ToolsRepoCommitish: 'main' + Variables: ${{ parameters.Variables }} + OperatingSystems: ${{ parameters.OperatingSystems }} + InstallLanguageSteps: ${{ parameters.InstallLanguageSteps }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Services: ${{ parameters.Services }} + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} + EnvVars: ${{ parameters.EnvVars}} From ec5fba827a49c80eaf37420eeef87c9406f8a092 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 8 Aug 2022 11:08:28 -0700 Subject: [PATCH 09/17] Remove unnecessary base yml --- .../pipelines/templates/jobs/perf-base.yml | 156 ------------------ eng/common/pipelines/templates/jobs/perf.yml | 136 +++++++++++++-- 2 files changed, 118 insertions(+), 174 deletions(-) delete mode 100644 eng/common/pipelines/templates/jobs/perf-base.yml diff --git a/eng/common/pipelines/templates/jobs/perf-base.yml b/eng/common/pipelines/templates/jobs/perf-base.yml deleted file mode 100644 index c9348f0a24..0000000000 --- a/eng/common/pipelines/templates/jobs/perf-base.yml +++ /dev/null @@ -1,156 +0,0 @@ -parameters: -- name: JobName - type: string - default: 'Perf' -- name: Language - type: string - default: '' -- name: LanguageRepoName - type: string - default: '' -- name: LanguageRepoCommitish - type: string - default: '' -- name: ToolsRepoCommitish - type: string - default: '' -- name: Variables - type: object - default: [] -- name: OperatingSystems - type: string - default: 'Linux' -- name: InstallLanguageSteps - type: stepList - default: [] -- name: ServiceDirectory - type: string - default: '' -- name: Services - type: string - default: '' -- name: PackageVersions - type: string - default: '.*' -- name: Tests - type: string - default: '.*' -- name: Arguments - type: string - default: '.*' -- name: Iterations - type: number - default: '5' -- name: AdditionalArguments - type: string - default: '' -- name: EnvVars - type: object - default: {} - -jobs: -- job: ${{ parameters.JobName }} - timeoutInMinutes: 360 - strategy: - matrix: - ${{ if contains(parameters.OperatingSystems, 'Linux') }}: - Linux: - Pool: 'azsdk-pool-mms-ubuntu-2004-perf' - OsVmImage: 'MMSUbuntu20.04' - MatrixName: 'Linux' - ${{ if contains(parameters.OperatingSystems, 'Windows') }}: - Windows: - Pool: 'azsdk-pool-mms-win-2019-perf' - OsVmImage: 'MMS2019' - MatrixName: 'Windows' - variables: - - ${{ parameters.Variables }} - pool: - name: $(Pool) - vmImage: $(OSVmImage) - steps: - - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '/*' - - '!sdk/**/test-recordings/*' - - '!sdk/**/recordings/*' - - '!sdk/**/SessionRecords/*' - - '!sdk/**/session-records/*' - Repositories: - - Name: ${{ parameters.LanguageRepoName }} - Commitish: ${{ parameters.LanguageRepoCommitish }} - WorkingDirectory: $(System.DefaultWorkingDirectory) - - Name: Azure/azure-sdk-tools - Commitish: ${{ parameters.ToolsRepoCommitish }} - WorkingDirectory: $(System.DefaultWorkingDirectory)/azure-sdk-tools - - - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml - parameters: - AgentImage: $(OSVmImage) - - - ${{ parameters.InstallLanguageSteps }} - - - template: /eng/common/TestResources/deploy-test-resources.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - Location: westus - ResourceType: perf - - - pwsh: | - set-content -path config.yml -value "WorkingDirectories:" - add-content -path config.yml -value " ${{ parameters.Language }}: $(Agent.BuildDirectory)/s" - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation - displayName: Create config.yml - - - script: >- - dotnet run -- run - --no-sync - --languages ${{ parameters.Language }} - --services "${{ parameters.Services }}" - --package-versions "${{ parameters.PackageVersions }}" - --tests "${{ parameters.Tests }}" - --arguments "${{ parameters.Arguments }}" - --iterations ${{ parameters.Iterations }} - ${{ parameters.AdditionalArguments }} - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation - env: - ${{ each var in parameters.EnvVars }}: - ${{ var.key }}: ${{ var.value }} - displayName: Run perf tests - - - pwsh: | - get-content results.txt - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - displayName: Print results.txt - condition: always() - - - pwsh: | - get-content results.csv - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - displayName: Print results.csv - condition: always() - - - pwsh: | - get-content results.md - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - displayName: Print results.md - condition: always() - - - pwsh: | - get-content results.json - workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - displayName: Print results.json - condition: always() - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results - artifactName: results-${{ parameters.Language }}-$(MatrixName) - condition: always() - - - template: /eng/common/TestResources/remove-test-resources.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - ResourceType: perf diff --git a/eng/common/pipelines/templates/jobs/perf.yml b/eng/common/pipelines/templates/jobs/perf.yml index 505eae90a2..f2ddf176a8 100644 --- a/eng/common/pipelines/templates/jobs/perf.yml +++ b/eng/common/pipelines/templates/jobs/perf.yml @@ -1,7 +1,19 @@ parameters: +- name: JobName + type: string + default: 'Perf' - name: Language type: string default: '' +- name: LanguageRepoName + type: string + default: $(Build.Repository.Name) +- name: LanguageRepoCommitish + type: string + default: $(Build.SourceVersion) +- name: ToolsRepoCommitish + type: string + default: 'main' - name: Variables type: object default: [] @@ -36,21 +48,109 @@ parameters: type: object default: {} -extends: - template: perf-base.yml - parameters: - Language: ${{ parameters.Language }} - LanguageRepoName: $(Build.Repository.Name) - LanguageRepoCommitish: $(Build.SourceVersion) - ToolsRepoCommitish: 'main' - Variables: ${{ parameters.Variables }} - OperatingSystems: ${{ parameters.OperatingSystems }} - InstallLanguageSteps: ${{ parameters.InstallLanguageSteps }} - ServiceDirectory: ${{ parameters.ServiceDirectory }} - Services: ${{ parameters.Services }} - PackageVersions: ${{ parameters.PackageVersions }} - Tests: ${{ parameters.Tests }} - Arguments: ${{ parameters.Arguments }} - Iterations: ${{ parameters.Iterations }} - AdditionalArguments: ${{ parameters.AdditionalArguments }} - EnvVars: ${{ parameters.EnvVars}} +jobs: +- job: ${{ parameters.JobName }} + timeoutInMinutes: 360 + strategy: + matrix: + ${{ if contains(parameters.OperatingSystems, 'Linux') }}: + Linux: + Pool: 'azsdk-pool-mms-ubuntu-2004-perf' + OsVmImage: 'MMSUbuntu20.04' + MatrixName: 'Linux' + ${{ if contains(parameters.OperatingSystems, 'Windows') }}: + Windows: + Pool: 'azsdk-pool-mms-win-2019-perf' + OsVmImage: 'MMS2019' + MatrixName: 'Windows' + variables: + - ${{ parameters.Variables }} + pool: + name: $(Pool) + vmImage: $(OSVmImage) + steps: + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - '/*' + - '!sdk/**/test-recordings/*' + - '!sdk/**/recordings/*' + - '!sdk/**/SessionRecords/*' + - '!sdk/**/session-records/*' + Repositories: + - Name: ${{ parameters.LanguageRepoName }} + Commitish: ${{ parameters.LanguageRepoCommitish }} + WorkingDirectory: $(System.DefaultWorkingDirectory) + - Name: Azure/azure-sdk-tools + Commitish: ${{ parameters.ToolsRepoCommitish }} + WorkingDirectory: $(System.DefaultWorkingDirectory)/azure-sdk-tools + + - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml + parameters: + AgentImage: $(OSVmImage) + + - ${{ parameters.InstallLanguageSteps }} + + - template: /eng/common/TestResources/deploy-test-resources.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Location: westus + ResourceType: perf + + - pwsh: | + set-content -path config.yml -value "WorkingDirectories:" + add-content -path config.yml -value " ${{ parameters.Language }}: $(Agent.BuildDirectory)/s" + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation + displayName: Create config.yml + + - script: >- + dotnet run -- run + --no-sync + --languages ${{ parameters.Language }} + --services "${{ parameters.Services }}" + --package-versions "${{ parameters.PackageVersions }}" + --tests "${{ parameters.Tests }}" + --arguments "${{ parameters.Arguments }}" + --iterations ${{ parameters.Iterations }} + ${{ parameters.AdditionalArguments }} + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation + env: + ${{ each var in parameters.EnvVars }}: + ${{ var.key }}: ${{ var.value }} + displayName: Run perf tests + + - pwsh: | + get-content results.txt + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results + displayName: Print results.txt + condition: always() + + - pwsh: | + get-content results.csv + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results + displayName: Print results.csv + condition: always() + + - pwsh: | + get-content results.md + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results + displayName: Print results.md + condition: always() + + - pwsh: | + get-content results.json + workingDirectory: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results + displayName: Print results.json + condition: always() + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: azure-sdk-tools/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/results + artifactName: results-${{ parameters.Language }}-$(MatrixName) + condition: always() + + - template: /eng/common/TestResources/remove-test-resources.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + ResourceType: perf From 8acffa12ebdbb23ce8b36bdab13c1bc7d1fd6bc9 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 8 Aug 2022 16:22:05 -0700 Subject: [PATCH 10/17] Fetch pull request refs --- eng/common/pipelines/templates/steps/sparse-checkout.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index a3b553b3a7..4106f1f2f2 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -39,7 +39,7 @@ steps: Write-Host "Repository $($repository.Name) is being initialized." Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." - git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) . + git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+refs/pull/*:refs/remotes/pull/*' https://github.com/$($repository.Name) . Write-Host "git sparse-checkout init" git sparse-checkout init From fbbd3590f9d1958064ad29d26587d9626b890421 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 22 Aug 2022 22:41:59 -0700 Subject: [PATCH 11/17] Fix refs path --- eng/common/pipelines/templates/steps/sparse-checkout.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index 4106f1f2f2..8a3c6009a3 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -39,7 +39,7 @@ steps: Write-Host "Repository $($repository.Name) is being initialized." Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." - git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+refs/pull/*:refs/remotes/pull/*' https://github.com/$($repository.Name) . + git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+refs/pull/*:refs/pull/*' https://github.com/$($repository.Name) . Write-Host "git sparse-checkout init" git sparse-checkout init From 7b3e8f8f3a215081a450d3db730bbfb6f9f61ac5 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Tue, 23 Aug 2022 20:01:47 -0700 Subject: [PATCH 12/17] Fetch commitish if matches PR syntax --- eng/common/pipelines/templates/steps/sparse-checkout.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index 8a3c6009a3..c6dd8ce3be 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -39,7 +39,7 @@ steps: Write-Host "Repository $($repository.Name) is being initialized." Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." - git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+refs/pull/*:refs/pull/*' https://github.com/$($repository.Name) . + git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) . Write-Host "git sparse-checkout init" git sparse-checkout init @@ -61,6 +61,9 @@ steps: # sparse-checkout commands after initial checkout will auto-checkout again if (!$hasInitialized) { + if ($repository.Commitish -match '^refs/pull/\d+/merge$') { + git fetch origin $($repository.Commitish):$($repository.Commitish) + } Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)" # This will use the default branch if repo.Commitish is empty git -c advice.detachedHead=false checkout $($repository.Commitish) From 4c3a3272295ecebde06295fdd81b22ae6eaa4cc4 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Tue, 23 Aug 2022 20:03:24 -0700 Subject: [PATCH 13/17] Print command before running --- eng/common/pipelines/templates/steps/sparse-checkout.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index c6dd8ce3be..d92cfbab67 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -62,8 +62,10 @@ steps: # sparse-checkout commands after initial checkout will auto-checkout again if (!$hasInitialized) { if ($repository.Commitish -match '^refs/pull/\d+/merge$') { + Write-Host "git fetch origin $($repository.Commitish):$($repository.Commitish)" git fetch origin $($repository.Commitish):$($repository.Commitish) } + Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)" # This will use the default branch if repo.Commitish is empty git -c advice.detachedHead=false checkout $($repository.Commitish) From bd356b1384f8b549cc1c12375968ccb5188accb3 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Tue, 23 Aug 2022 22:08:23 -0700 Subject: [PATCH 14/17] Add PR commitish to remote.origin.fetch --- .../pipelines/templates/steps/sparse-checkout.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index d92cfbab67..49715bbeff 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -38,8 +38,13 @@ steps: } else { Write-Host "Repository $($repository.Name) is being initialized." - Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." - git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) . + if ($repository.Commitish -match '^refs/pull/\d+/merge$') { + Write-Host "git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+$($repository.Commitish):$($repository.Commitish)' https://github.com/$($repository.Name) ." + git clone --no-checkout --filter=tree:0 -c remote.origin.fetch=''+$($repository.Commitish):$($repository.Commitish)'' https://github.com/$($repository.Name) . + } else { + Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." + git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) . + } Write-Host "git sparse-checkout init" git sparse-checkout init @@ -61,11 +66,6 @@ steps: # sparse-checkout commands after initial checkout will auto-checkout again if (!$hasInitialized) { - if ($repository.Commitish -match '^refs/pull/\d+/merge$') { - Write-Host "git fetch origin $($repository.Commitish):$($repository.Commitish)" - git fetch origin $($repository.Commitish):$($repository.Commitish) - } - Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)" # This will use the default branch if repo.Commitish is empty git -c advice.detachedHead=false checkout $($repository.Commitish) From 23a50d7cca28ea9b11ada20a8dc31b62a9c00eda Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Wed, 24 Aug 2022 00:35:38 -0700 Subject: [PATCH 15/17] Add pool and vmimage parameters --- eng/common/pipelines/templates/jobs/perf.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/perf.yml b/eng/common/pipelines/templates/jobs/perf.yml index f2ddf176a8..ecd794a1f6 100644 --- a/eng/common/pipelines/templates/jobs/perf.yml +++ b/eng/common/pipelines/templates/jobs/perf.yml @@ -2,6 +2,18 @@ parameters: - name: JobName type: string default: 'Perf' +- name: LinuxPool + type: string + default: 'azsdk-pool-mms-ubuntu-2004-perf' +- name: LinuxVmImage + type: string + default: 'MMSUbuntu20.04' +- name: WindowsPool + type: string + default: 'azsdk-pool-mms-win-2019-perf' +- name: WindowsVmImage + type: string + default: 'MMS2019' - name: Language type: string default: '' @@ -55,13 +67,13 @@ jobs: matrix: ${{ if contains(parameters.OperatingSystems, 'Linux') }}: Linux: - Pool: 'azsdk-pool-mms-ubuntu-2004-perf' - OsVmImage: 'MMSUbuntu20.04' + Pool: ${{ parameters.LinuxPool }} + OsVmImage: ${{ parameters.LinuxVmImage }} MatrixName: 'Linux' ${{ if contains(parameters.OperatingSystems, 'Windows') }}: Windows: - Pool: 'azsdk-pool-mms-win-2019-perf' - OsVmImage: 'MMS2019' + Pool: ${{ parameters.WindowsPool }} + OsVmImage: ${{ parameters.WindowsVmImage }} MatrixName: 'Windows' variables: - ${{ parameters.Variables }} From 71c968fb2b05e64e46ba28d68f37a31bd8e247c4 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Thu, 25 Aug 2022 12:20:32 -0700 Subject: [PATCH 16/17] Map ref under "refs/remotes/origin" --- eng/common/pipelines/templates/steps/sparse-checkout.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index 49715bbeff..50169ec652 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -39,7 +39,7 @@ steps: Write-Host "Repository $($repository.Name) is being initialized." if ($repository.Commitish -match '^refs/pull/\d+/merge$') { - Write-Host "git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+$($repository.Commitish):$($repository.Commitish)' https://github.com/$($repository.Name) ." + Write-Host "git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+$($repository.Commitish):refs/remotes/origin/$($repository.Commitish)' https://github.com/$($repository.Name) ." git clone --no-checkout --filter=tree:0 -c remote.origin.fetch=''+$($repository.Commitish):$($repository.Commitish)'' https://github.com/$($repository.Name) . } else { Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." From cf260bd5f857ef980da108a070f4b46015231f2e Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Thu, 25 Aug 2022 12:32:03 -0700 Subject: [PATCH 17/17] Add ref to refs/remotes/origin --- eng/common/pipelines/templates/steps/sparse-checkout.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index 50169ec652..0a2b537be1 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -40,7 +40,7 @@ steps: if ($repository.Commitish -match '^refs/pull/\d+/merge$') { Write-Host "git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+$($repository.Commitish):refs/remotes/origin/$($repository.Commitish)' https://github.com/$($repository.Name) ." - git clone --no-checkout --filter=tree:0 -c remote.origin.fetch=''+$($repository.Commitish):$($repository.Commitish)'' https://github.com/$($repository.Name) . + git clone --no-checkout --filter=tree:0 -c remote.origin.fetch=''+$($repository.Commitish):refs/remotes/origin/$($repository.Commitish)'' https://github.com/$($repository.Name) . } else { Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) .