diff --git a/eng/pipelines/common/perf-variables.yml b/eng/pipelines/common/perf-variables.yml deleted file mode 100644 index 9b14ab5067a05c..00000000000000 --- a/eng/pipelines/common/perf-variables.yml +++ /dev/null @@ -1,16 +0,0 @@ -variables: - - name: _wasmCollectHelixLogsScript - value: >- - test -d "$HELIX_WORKITEM_UPLOAD_ROOT" && ( - export _PERF_DIR=$HELIX_WORKITEM_ROOT/performance; - mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log; - find $_PERF_DIR -name '*.binlog' | xargs -I{} cp {} $HELIX_WORKITEM_UPLOAD_ROOT/log; - test "$_commandExitCode" -eq 0 || ( - mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/obj; - mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/bin; - mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/BenchmarkDotNet.Autogenerated/obj; - mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/for-running; - cp -R $_PERF_DIR/artifacts/obj/MicroBenchmarks $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/obj; - cp -R $_PERF_DIR/artifacts/bin/MicroBenchmarks $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/bin; - cp -R $_PERF_DIR/artifacts/obj/BenchmarkDotNet.Autogenerated $HELIX_WORKITEM_UPLOAD_ROOT/log/BenchmarkDotNet.Autogenerated/obj; - cp -R $_PERF_DIR/artifacts/bin/for-running $HELIX_WORKITEM_UPLOAD_ROOT/log/for-running)) diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml index be6ee86df631e4..63b9296e0cbb79 100644 --- a/eng/pipelines/common/variables.yml +++ b/eng/pipelines/common/variables.yml @@ -58,6 +58,4 @@ variables: ne(variables['isExtraPlatformsBuild'], true), eq(variables['isRollingBuild'], true))) ] -- template: /eng/pipelines/common/perf-variables.yml - -- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml \ No newline at end of file +- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml diff --git a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml index 8419e99f7ba80b..f750bda72f92eb 100644 --- a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml +++ b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml @@ -1,3 +1,6 @@ +parameters: + perfBranch: 'main' + jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'Schedule')) }}: @@ -100,6 +103,7 @@ jobs: - template: /eng/pipelines/coreclr/templates/build-and-run-perf-ios-scenarios.yml parameters: hybridGlobalization: True + perfBranch: ${{ parameters.perfBranch }} # run android scenarios - template: /eng/pipelines/common/platform-matrix.yml @@ -112,10 +116,11 @@ jobs: jobParameters: testGroup: perf runtimeType: AndroidMono - projectFile: android_scenarios.proj + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/android_scenarios.proj runKind: android_scenarios runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perfpixel4a' + perfBranch: ${{ parameters.perfBranch }} # run mono microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -129,10 +134,11 @@ jobs: testGroup: perf liveLibrariesBuildConfig: Release runtimeType: mono - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro_mono runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' + perfBranch: ${{ parameters.perfBranch }} # run mono interpreter perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -147,10 +153,11 @@ jobs: liveLibrariesBuildConfig: Release runtimeType: mono codeGenType: 'Interpreter' - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro_mono runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' + perfBranch: ${{ parameters.perfBranch }} # run mono aot microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -165,10 +172,11 @@ jobs: liveLibrariesBuildConfig: Release runtimeType: mono codeGenType: 'AOT' - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro_mono runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' + perfBranch: ${{ parameters.perfBranch }} # run coreclr perftiger microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -184,10 +192,11 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' + perfBranch: ${{ parameters.perfBranch }} # run coreclr perftiger microbenchmarks no dynamic pgo perf jobs - template: /eng/pipelines/common/platform-matrix.yml @@ -200,11 +209,12 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' pgoRunType: -NoDynamicPGO + perfBranch: ${{ parameters.perfBranch }} - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -216,11 +226,12 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' pgoRunType: --nodynamicpgo + perfBranch: ${{ parameters.perfBranch }} # run coreclr perftiger microbenchmarks no R2R perf jobs - template: /eng/pipelines/common/platform-matrix.yml @@ -233,11 +244,12 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' r2rRunType: -NoR2R + perfBranch: ${{ parameters.perfBranch }} - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -249,11 +261,12 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' r2rRunType: --nor2r + perfBranch: ${{ parameters.perfBranch }} # run coreclr perfowl microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -267,10 +280,11 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfowl' + perfBranch: ${{ parameters.perfBranch }} # run coreclr perfviper microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -284,10 +298,11 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfviper' + perfBranch: ${{ parameters.perfBranch }} # run coreclr perfviper microbenchmarks perf jitoptrepeat jobs - template: /eng/pipelines/common/platform-matrix.yml @@ -301,11 +316,12 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfviper' experimentName: 'jitoptrepeat' + perfBranch: ${{ parameters.perfBranch }} # run coreclr crossgen perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -319,10 +335,11 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: crossgen_perf.proj + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj runKind: crossgen_scenarios runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perftiger_crossgen' + perfBranch: ${{ parameters.perfBranch }} # build mono runtime packs - template: /eng/pipelines/common/platform-matrix.yml @@ -368,3 +385,4 @@ jobs: archiveExtension: '.tar.gz' archiveType: tar tarCompression: gz + perfBranch: ${{ parameters.perfBranch }} diff --git a/eng/pipelines/coreclr/perf-wasm-jobs.yml b/eng/pipelines/coreclr/perf-wasm-jobs.yml index 3f33e90cbc8f1d..93ffdfc5283821 100644 --- a/eng/pipelines/coreclr/perf-wasm-jobs.yml +++ b/eng/pipelines/coreclr/perf-wasm-jobs.yml @@ -7,7 +7,7 @@ parameters: onlySanityCheck: false downloadSpecificBuild: null # { buildId, pipeline, branchName, project } collectHelixLogsScript: '' - perfForkToUse: {} # url, branch + perfBranch: 'main' jobs: @@ -43,14 +43,14 @@ jobs: liveLibrariesBuildConfig: Release runtimeType: wasm codeGenType: 'wasm' - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' javascriptEngine: 'javascriptcore' collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }} downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} - perfForkToUse: ${{ parameters.perfForkToUse }} + perfBranch: ${{ parameters.perfBranch }} #downloadSpecificBuild: #buildId: '1693181' #pipeline: 'perf-wasm' @@ -69,14 +69,14 @@ jobs: livelibrariesbuildconfig: Release runtimetype: wasm codegentype: 'aot' - projectfile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runkind: micro runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' javascriptengine: 'javascriptcore' collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }} downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} - perfForkToUse: ${{ parameters.perfForkToUse }} + perfBranch: ${{ parameters.perfBranch }} - ${{ if eq(parameters.runProfile, 'v8') }}: - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: @@ -110,7 +110,7 @@ jobs: liveLibrariesBuildConfig: Release runtimeType: wasm codeGenType: 'wasm' - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' @@ -121,7 +121,7 @@ jobs: compare: ${{ parameters.compare }} onlySanityCheck: ${{ parameters.onlySanityCheck }} downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} - perfForkToUse: ${{ parameters.perfForkToUse }} + perfBranch: ${{ parameters.perfBranch }} #run mono wasm aot microbenchmarks perf job # Disabled for runtime-wasm-perf on PRs due to https://github.com/dotnet/runtime/issues/95101 @@ -138,7 +138,7 @@ jobs: livelibrariesbuildconfig: Release runtimetype: wasm codegentype: 'aot' - projectfile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runkind: micro runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perftiger' @@ -149,7 +149,7 @@ jobs: compare: ${{ parameters.compare }} onlySanityCheck: ${{ parameters.onlySanityCheck }} downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} - perfForkToUse: ${{ parameters.perfForkToUse }} + perfBranch: ${{ parameters.perfBranch }} # run mono wasm blazor perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -163,14 +163,14 @@ jobs: testGroup: perf liveLibrariesBuildConfig: Release runtimeType: wasm - projectFile: blazor_perf.proj + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/blazor_perf.proj runKind: blazor_scenarios runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm) - #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to performance-setup.sh + #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to run-performance-job.py logicalmachine: 'perftiger' downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} - perfForkToUse: ${{ parameters.perfForkToUse }} + perfBranch: ${{ parameters.perfBranch }} - ${{if or(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), in(variables['Build.DefinitionName'], 'runtime-wasm-perf')) }}: # run mono wasm blazor perf job @@ -185,12 +185,12 @@ jobs: testGroup: perf liveLibrariesBuildConfig: Release runtimeType: wasm - projectFile: blazor_perf.proj + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/blazor_perf.proj runKind: blazor_scenarios runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm) - #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to performance-setup.sh + #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to run-performance-job.py logicalmachine: 'perftiger' downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} - perfForkToUse: ${{ parameters.perfForkToUse }} hybridGlobalization: True + perfBranch: ${{ parameters.perfBranch }} diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index 3d7ee8106709e7..9b595f3dd8c1e7 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -1,3 +1,9 @@ +parameters: +- name: perfBranch + displayName: Performance Repo Branch + type: string + default: 'main' + trigger: batch: true branches: @@ -19,7 +25,7 @@ trigger: - THIRD-PARTY-NOTICES.TXT variables: - - template: /eng/pipelines/common/variables.yml +- template: /eng/pipelines/common/variables.yml # # For the 'schedule' case, only wasm/jsc perf jobs are run. @@ -46,9 +52,12 @@ extends: - template: /eng/pipelines/coreclr/perf-wasm-jobs.yml parameters: collectHelixLogsScript: ${{ variables._wasmCollectHelixLogsScript }} + perfBranch: ${{ parameters.perfBranch }} #${{ and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')) }}: # runProfile: 'non-v8' ${{ if ne(variables['System.TeamProject'], 'public') }}: runProfile: 'v8' - template: /eng/pipelines/coreclr/perf-non-wasm-jobs.yml + parameters: + perfBranch: ${{ parameters.perfBranch }} diff --git a/eng/pipelines/coreclr/perf_slow.yml b/eng/pipelines/coreclr/perf_slow.yml index 2de43e9e8181fd..e7d073534a722b 100644 --- a/eng/pipelines/coreclr/perf_slow.yml +++ b/eng/pipelines/coreclr/perf_slow.yml @@ -5,6 +5,10 @@ parameters: - name: runScheduledJobs type: boolean default: false +- name: perfBranch + displayName: Performance Repo Branch + type: string + default: 'main' trigger: batch: true @@ -80,11 +84,12 @@ extends: liveLibrariesBuildConfig: Release runtimeType: mono codeGenType: 'Interpreter' - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro_mono runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfampere' timeoutInMinutes: 720 + perfBranch: ${{ parameters.perfBranch }} # run arm64 jit jobs for mono - template: /eng/pipelines/common/platform-matrix.yml @@ -98,11 +103,12 @@ extends: testGroup: perf liveLibrariesBuildConfig: Release runtimeType: mono - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro_mono runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfampere' timeoutInMinutes: 720 + perfBranch: ${{ parameters.perfBranch }} - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(notin(variables['Build.Reason'], 'Schedule', 'Manual'), parameters.runPrivateJobs)) }}: @@ -167,11 +173,12 @@ extends: liveLibrariesBuildConfig: Release runtimeType: mono codeGenType: 'AOT' - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro_mono runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfampere' timeoutInMinutes: 780 + perfBranch: ${{ parameters.perfBranch }} # run coreclr Linux arm64 ampere microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -184,11 +191,12 @@ extends: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfampere' timeoutInMinutes: 780 + perfBranch: ${{ parameters.perfBranch }} #run coreclr Linux arm64 ampere no dynamic pgo microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -201,12 +209,13 @@ extends: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfampere' timeoutInMinutes: 780 pgoRunType: --nodynamicpgo + perfBranch: ${{ parameters.perfBranch }} #run coreclr Linux arm64 ampere no R2R microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -219,12 +228,13 @@ extends: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfampere' timeoutInMinutes: 780 r2rRunType: --nor2r + perfBranch: ${{ parameters.perfBranch }} # run coreclr Windows arm64 microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -237,10 +247,11 @@ extends: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfsurf' + perfBranch: ${{ parameters.perfBranch }} # run coreclr Windows arm64 ampere microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -253,11 +264,12 @@ extends: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfampere' timeoutInMinutes: 780 + perfBranch: ${{ parameters.perfBranch }} # run coreclr Windows arm64 ampere no dynamic pgo microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -270,12 +282,13 @@ extends: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfampere' pgoRunType: -NoDynamicPGO timeoutInMinutes: 780 + perfBranch: ${{ parameters.perfBranch }} # run coreclr Windows arm64 ampere no R2R microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -288,12 +301,13 @@ extends: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'perfampere' r2rRunType: -NoR2R timeoutInMinutes: 780 + perfBranch: ${{ parameters.perfBranch }} # run coreclr cloudvm microbenchmarks perf job # this run is added temporarily for measuring AVX-512 performance @@ -308,10 +322,11 @@ extends: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj + projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj runKind: micro runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml logicalmachine: 'cloudvm' + perfBranch: ${{ parameters.perfBranch }} # Uncomment once we fix https://github.com/dotnet/performance/issues/1950 # # run coreclr linux crossgen perf job @@ -325,10 +340,11 @@ extends: # jobParameters: # testGroup: perf # liveLibrariesBuildConfig: Release - # projectFile: crossgen_perf.proj + # projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj # runKind: crossgen_scenarios # runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml # logicalmachine: 'perfa64' + # perfBranch: ${{ parameters.perfBranch }} # # run coreclr windows crossgen perf job # - template: /eng/pipelines/common/platform-matrix.yml @@ -341,7 +357,8 @@ extends: # jobParameters: # testGroup: perf # liveLibrariesBuildConfig: Release - # projectFile: crossgen_perf.proj + # projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj # runKind: crossgen_scenarios # runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml # logicalmachine: 'perfsurf' + # perfBranch: ${{ parameters.perfBranch }} diff --git a/eng/pipelines/coreclr/templates/build-and-run-perf-ios-scenarios.yml b/eng/pipelines/coreclr/templates/build-and-run-perf-ios-scenarios.yml index 2d484c8e15e92d..f9513e556f87e0 100644 --- a/eng/pipelines/coreclr/templates/build-and-run-perf-ios-scenarios.yml +++ b/eng/pipelines/coreclr/templates/build-and-run-perf-ios-scenarios.yml @@ -1,5 +1,6 @@ parameters: hybridGlobalization: True + perfBranch: 'main' jobs: # build mono iOS scenarios HybridGlobalization @@ -61,13 +62,14 @@ jobs: jobParameters: testGroup: perf runtimeType: iOSMono - projectFile: ios_scenarios.proj + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perfiphone12mini' iOSLlvmBuild: False iOSStripSymbols: False hybridGlobalization: ${{ parameters.hybridGlobalization }} + perfBranch: ${{ parameters.perfBranch }} - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -79,13 +81,14 @@ jobs: jobParameters: testGroup: perf runtimeType: iOSMono - projectFile: ios_scenarios.proj + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perfiphone12mini' iOSLlvmBuild: False iOSStripSymbols: True hybridGlobalization: ${{ parameters.hybridGlobalization }} + perfBranch: ${{ parameters.perfBranch }} - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -97,13 +100,14 @@ jobs: jobParameters: testGroup: perf runtimeType: iOSMono - projectFile: ios_scenarios.proj + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perfiphone12mini' iOSLlvmBuild: True iOSStripSymbols: False hybridGlobalization: ${{ parameters.hybridGlobalization }} + perfBranch: ${{ parameters.perfBranch }} - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -115,13 +119,14 @@ jobs: jobParameters: testGroup: perf runtimeType: iOSMono - projectFile: ios_scenarios.proj + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perfiphone12mini' iOSLlvmBuild: True iOSStripSymbols: True hybridGlobalization: ${{ parameters.hybridGlobalization }} + perfBranch: ${{ parameters.perfBranch }} # run NativeAOT iOS scenarios HybridGlobalization - template: /eng/pipelines/common/platform-matrix.yml @@ -134,12 +139,13 @@ jobs: jobParameters: testGroup: perf runtimeType: iOSNativeAOT - projectFile: ios_scenarios.proj + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perfiphone12mini' iOSStripSymbols: False hybridGlobalization: ${{ parameters.hybridGlobalization }} + perfBranch: ${{ parameters.perfBranch }} - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -151,9 +157,10 @@ jobs: jobParameters: testGroup: perf runtimeType: iOSNativeAOT - projectFile: ios_scenarios.proj + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perfiphone12mini' iOSStripSymbols: True hybridGlobalization: ${{ parameters.hybridGlobalization }} + perfBranch: ${{ parameters.perfBranch }} diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 3b0c166b2e51d5..daaaaad0753b21 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -29,8 +29,8 @@ parameters: compare: false onlySanityCheck: false downloadSpecificBuild: null # buildId, pipeline, branchName, project - perfForkToUse: {} # url, branch crossBuild: false + perfBranch: 'main' ### Perf job @@ -66,6 +66,7 @@ jobs: iosStripSymbols: ${{ parameters.iosStripSymbols }} hybridGlobalization: ${{ parameters.hybridGlobalization }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + extraSetupParameters: $(extraSetupParameters) ${{ if and(eq(parameters.runtimeType, 'wasm'), eq(parameters.codeGenType, 'aot')) }}: helixTypeSuffix: '/wasm/aot' @@ -93,52 +94,64 @@ jobs: - ${{ if eq(parameters.runtimeType, 'iOSNativeAOT')}}: - ${{ 'build_ios_arm64_release_iOSNativeAOT' }} - ${{ if and(eq(parameters.osGroup, 'windows'), not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) }}: - ${{ if eq(parameters.runtimeType, 'mono') }}: - extraSetupParameters: -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono - ${{ if eq(parameters.runtimeType, 'coreclr') }}: - extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} - ${{ if and(ne(parameters.osGroup, 'windows'), not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) }}: - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: - extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono - ${{ if and(eq(parameters.runtimeType, 'wasm'), ne(parameters.codeGenType, 'AOT')) }}: - extraSetupParameters: --architecture ${{ parameters.archType }} --wasmbundle $(librariesDownloadDir)/bin/wasm --javascriptengine ${{ parameters.javascriptEngine }} --hybridglobalization ${{ parameters.hybridGlobalization }} $(extraSetupParametersSuffix) - ${{ if and(eq(parameters.runtimeType, 'wasm'), eq(parameters.codeGenType, 'AOT')) }}: - extraSetupParameters: --architecture ${{ parameters.archType }} --wasmbundle $(librariesDownloadDir)/bin/wasm --wasmaot --javascriptengine ${{ parameters.javascriptEngine }} $(extraSetupParametersSuffix) - ${{ if and(eq(parameters.codeGenType, 'AOT'), ne(parameters.runtimeType, 'wasm')) }}: - extraSetupParameters: --architecture ${{ parameters.archType }} --monoaot $(librariesDownloadDir)/bin/aot --llvm - ${{ if and(eq(parameters.runtimeType, 'coreclr'), ne(parameters.osSubGroup, '_musl')) }}: - extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} - ${{ if and(eq(parameters.runtimeType, 'coreclr'), eq(parameters.osSubGroup, '_musl')) }}: - extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} --alpine - ${{ if in(parameters.runtimeType, 'AndroidMono') }}: - extraSetupParameters: -Architecture ${{ parameters.archType }} -AndroidMono - ${{ if in(parameters.runtimeType, 'iOSMono') }}: - extraSetupParameters: --architecture ${{ parameters.archType }} --iosmono --iosllvmbuild ${{ parameters.iOSLlvmBuild }} --iosstripsymbols ${{ parameters.iOSStripSymbols }} --hybridglobalization ${{ parameters.hybridGlobalization }} - ${{ if in(parameters.runtimeType, 'iOSNativeAOT') }}: - extraSetupParameters: --architecture ${{ parameters.archType }} --iosnativeaot --iosllvmbuild ${{ parameters.iOSLlvmBuild }} --iosstripsymbols ${{ parameters.iOSStripSymbols }} --hybridglobalization ${{ parameters.hybridGlobalization }} - variables: - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} - _crossBuildPropertyArg: '' + - name: _subGroupArg + ${{ if ne(parameters.osSubgroup, '') }}: + value: '--os-sub-group ${{ parameters.osSubgroup }}' - name: _compareArg ${{ if eq(parameters.compare, 'true') }}: value: '--compare' - name: _onlySanityCheckArg ${{ if eq(parameters.onlySanityCheck, 'true') }}: value: '--only-sanity' - - name: _perfForkArg - ${{ if ne(parameters.perfForkToUse.url, '') }}: - value: --perffork ${{ parameters.perfForkToUse.url }} --perfforkbranch ${{ coalesce(parameters.perfForkToUse.branch, 'main') }} + - name: _hybridGlobalizationArg + ${{ if eq(parameters.hybridGlobalization, 'True') }}: + value: '--hybrid-globalization' + - name: _pgoRunTypeArg + ${{ if ne(parameters.pgoRunType, '') }}: + value: '--pgo-run-type ${{ parameters.pgoRunType }}' + - name: _physicalPromotionRunTypeArg + ${{ if ne(parameters.physicalPromotionRunType, '') }}: + value: '--physical-promotion-run-type ${{ parameters.physicalPromotionRunType }}' + - name: _r2rRunTypeArg + ${{ if ne(parameters.r2rRunType, '') }}: + value: '--r2r-run-type ${{ parameters.r2rRunType }}' + - name: _experimentArg + ${{ if ne(parameters.experimentName, '') }}: + value: '--experiment-name ${{ parameters.experimentName }}' + - name: _jsEngineArg + ${{ if eq(parameters.runtimeType, 'wasm') }}: + value: '--javascript-engine ${{ parameters.javascriptEngine }}' + - name: _iOSLlvmBuildArg + ${{ if eq(parameters.iOSLlvmBuild, 'True') }}: + value: '--ios-llvm-build' + - name: _iOSStripSymbolsArg + ${{ if eq(parameters.iOSStripSymbols, 'True') }}: + value: '--ios-strip-symbols' + - name: _iosArgs + ${{ if in(parameters.runtimeType, 'iOSMono', 'iOSNativeAOT') }}: + value: '$(_iOSLlvmBuildArg) $(_iOSStripSymbolsArg)' + - name: _projectFileArg + ${{ if ne(parameters.projectFile, '') }}: + value: '--project-file ${{ parameters.projectFile }}' - - name: extraSetupParametersSuffix - value: "$(_compareArg) $(_onlySanityCheckArg) $(_perfForkArg)" + - name: extraSetupParameters + value: "--run-kind ${{ parameters.runKind }} --architecture ${{ parameters.archType }} --os-group ${{ parameters.osGroup }} $(_subGroupArg) --logical-machine ${{ parameters.logicalMachine }} --performance-repo-dir $(Agent.BuildDirectory)/performance --runtime-repo-dir $(Build.SourcesDirectory) --codegen-type ${{ parameters.codeGenType }} --runtime-type ${{ parameters.runtimeType }} --use-local-commit-time $(_compareArg) $(_onlySanityCheckArg) $(_hybridGlobalizationArg) $(_pgoRunTypeArg) $(_physicalPromotionRunTypeArg) $(_r2rRunTypeArg) $(_experimentArg) $(_jsEngineArg) $(_iosArgs) $(_projectFileArg)" - ${{ if eq(parameters.crossBuild, true) }}: - _crossBuildPropertyArg: '-cross' + - ${{ if eq(parameters.osGroup, 'windows') }}: + - name: Python + value: 'py -3' + - ${{ if ne(parameters.osGroup, 'windows') }}: + - name: Python + value: 'python3' + - name: librariesDownloadDir value: '$(Build.SourcesDirectory)/artifacts' @@ -151,6 +164,25 @@ jobs: frameworks: - ${{ parameters.framework }} steps: + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - script: | + set -x + pwd + git clone https://github.com/dotnet/performance.git --depth 1 -b ${{ parameters.perfBranch }} --single-branch + displayName: Clone performance + workingDirectory: $(Agent.BuildDirectory) + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + # Copy dotnet-runtime directory to Build.SourcesDirectory (TODO: Update yaml to just use the dotnet-runtime directory) + - task: CopyFiles@2 + inputs: + SourceFolder: $(Build.SourcesDirectory)/dotnet-runtime + TargetFolder: $(Build.SourcesDirectory) + # And remove the original + - task: DeleteFiles@1 + inputs: + SourceFolder: $(Build.SourcesDirectory)/dotnet-runtime + - checkout: git://internal/dotnet-performance@${{ parameters.perfBranch }} + path: performance # Extra steps that will be passed to the performance template and run before sending the job to helix (all of which is done in the template) - script: | $(Build.SourcesDirectory)/eng/common/msbuild.sh $(Build.SourcesDirectory)/eng/testing/performance/add_properties_to_pipeline.proj /t:SetVariables diff --git a/eng/pipelines/coreclr/templates/perf-send-to-helix.yml b/eng/pipelines/coreclr/templates/perf-send-to-helix.yml index c11b0ffe8b76ff..f021d666292a87 100644 --- a/eng/pipelines/coreclr/templates/perf-send-to-helix.yml +++ b/eng/pipelines/coreclr/templates/perf-send-to-helix.yml @@ -1,53 +1,18 @@ -# Please remember to update the documentation if you make changes to these parameters! parameters: ProjectFile: '' # required -- project file that specifies the helix workitems - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases.json - EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - Creator: '' # optional -- if the build is external, use this to specify who is sending the job + osGroup: '' # required -- operating system for the job DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - osGroup: '' # required -- operating system for the job - HelixPrereqCommands: '' steps: - template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml parameters: osGroup: ${{ parameters.osGroup }} - sendParams: $(Build.SourcesDirectory)/eng/testing/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog + sendParams: ${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} condition: ${{ parameters.condition }} shouldContinueOnError: ${{ parameters.continueOnError }} environment: BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - HelixPrereqCommands: ${{ parameters.HelixPrereqCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml index 1feb524a2deba4..d824377b275256 100644 --- a/eng/pipelines/coreclr/templates/run-performance-job.yml +++ b/eng/pipelines/coreclr/templates/run-performance-job.yml @@ -22,12 +22,11 @@ parameters: experimentName: '' # optional -- name of the experiment runtimeType: 'coreclr' # optional -- Sets the runtime as coreclr or mono codeGenType: 'JIT' # optional -- Decides on the codegen technology if running on mono - projectFile: 'microbenchmarks.proj' # optional -- project file to build helix workitems + projectFile: '$(Agent.BuildDirectory)/performance/eng/performance/helix.proj' # optional -- project file to build helix workitems runKind: '' # required -- test category logicalMachine: '' # required -- Used to specify a which pool of machines the test should run against javascriptEngine: 'NoJS' helixTypeSuffix: '' # optional -- appends to HelixType - collectHelixLogsScript: '' # optional -- script to collect the logs, and artifacts helpful for debugging failures additionalSetupParameters: '' # optional -- additional setup parameters that are job-specific jobs: @@ -57,108 +56,15 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} - - IsInternal: '' - - HelixApiAccessToken: '' - - ${{ if and(eq(parameters.runtimeType, 'wasm'), in(variables['Build.Reason'], 'PullRequest')) }}: - - HelixPerfUploadTokenValue: '' - - ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.osGroup, 'windows')) }}: - - HelixPerfUploadTokenValue: '$(PerfCommandUploadTokenLinux)' - - ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.osGroup, 'windows')) }}: - - HelixPerfUploadTokenValue: '$(PerfCommandUploadToken)' - - ${{ if eq(parameters.runtimeType, 'wasm') }}: - # nodejs installation steps from https://github.com/nodesource/distributions - - HelixPreCommandsWasmOnLinux: >- - export RestoreAdditionalProjectSources=$HELIX_CORRELATION_PAYLOAD/built-nugets && - sudo apt-get -y remove nodejs && - sudo apt-get update && - sudo apt-get install -y ca-certificates curl gnupg && - sudo mkdir -p /etc/apt/keyrings && - sudo rm -f /etc/apt/keyrings/nodesource.gpg && - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor --batch -o /etc/apt/keyrings/nodesource.gpg && - export NODE_MAJOR=18 && - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list && - sudo apt-get update && - sudo apt autoremove -y && - sudo apt-get install nodejs -y && - test -n "$(V8Version)" && - npm install --prefix $HELIX_WORKITEM_PAYLOAD jsvu -g && - $HELIX_WORKITEM_PAYLOAD/bin/jsvu --os=linux64 v8@$(V8Version) && - export V8_ENGINE_PATH=~/.jsvu/bin/v8-$(V8Version) && - ${V8_ENGINE_PATH} -e 'console.log(`V8 version: ${this.version()}`)' - - ${{ if ne(parameters.runtimeType, 'wasm') }}: - - HelixPreCommandsWasmOnLinux: echo - - HelixPreCommandStemWindows: 'set ORIGPYPATH=%PYTHONPATH%;py -m pip install -U pip;py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;echo on;set PYTHONPATH=;python -m pip install -U pip;python -m pip install urllib3==1.26.15;python -m pip install azure.storage.blob==12.13.0;python -m pip install azure.storage.queue==12.4.0;python -m pip install azure.identity==1.16.1;set "PERFLAB_UPLOAD_TOKEN=$(HelixPerfUploadTokenValue)"' - - HelixPreCommandStemLinux: >- - export ORIGPYPATH=$PYTHONPATH - export CRYPTOGRAPHY_ALLOW_OPENSSL_102=true; - echo "** Installing prerequistes **"; - echo "** Waiting for dpkg to unlock (up to 2 minutes) **" && - timeout 2m bash -c 'while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do if [ -z "$printed" ]; then echo "Waiting for dpkg lock to be released... Lock is held by: $(ps -o cmd= -p $(sudo fuser /var/lib/dpkg/lock-frontend))"; printed=1; fi; echo "Waiting 5 seconds to check again"; sleep 5; done;' && - sudo apt-get remove -y lttng-modules-dkms && - sudo apt-get -y install python3-pip && - python3 -m pip install --user -U pip && - python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv && - ls -l $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate && - export PYTHONPATH= && - python3 -m pip install --user -U pip && - pip3 install urllib3==1.26.15 && - pip3 install --user azure.storage.blob==12.13.0 && - pip3 install --user azure.storage.queue==12.4.0 && - pip3 install --user azure.identity==1.16.1 && - sudo apt-get update && - sudo apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates && - $(HelixPreCommandsWasmOnLinux) && - export PERFLAB_UPLOAD_TOKEN="$(HelixPerfUploadTokenValue)" - || export PERF_PREREQS_INSTALL_FAILED=1; - test "x$PERF_PREREQS_INSTALL_FAILED" = "x1" && echo "** Error: Failed to install prerequites **" - - HelixPreCommandStemMusl: 'ulimit -n 4096;export ORIGPYPATH=$PYTHONPATH;sudo apk add icu-libs krb5-libs libgcc libintl libssl1.1 libstdc++ zlib cargo;sudo apk add libgdiplus --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing; python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install urllib3==1.26.15;pip3 install azure.storage.blob==12.13.0;pip3 install azure.storage.queue==12.4.0;pip3 install azure.identity==1.16.1;export PERFLAB_UPLOAD_TOKEN="$(HelixPerfUploadTokenValue)"' - - ExtraMSBuildLogsWindows: 'set MSBUILDDEBUGCOMM=1;set "MSBUILDDEBUGPATH=%HELIX_WORKITEM_UPLOAD_ROOT%"' - - ExtraMSBuildLogsLinux: 'export MSBUILDDEBUGCOMM=1;export "MSBUILDDEBUGPATH=$HELIX_WORKITEM_UPLOAD_ROOT"' - - HelixPreCommand: '' - - HelixPostCommand: '' - - Interpreter: '' + - name: internalParam + value: '' - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq( parameters.osGroup, 'windows') }}: - - HelixPreCommand: $(HelixPreCommandStemWindows);$(ExtraMSBuildLogsWindows) - - HelixPostCommand: 'set PYTHONPATH=%ORIGPYPATH%' - - IsInternal: -Internal - - ${{ if ne(parameters.osGroup, 'windows') }}: - - ${{ if eq(parameters.osSubGroup, '_musl') }}: - - HelixPreCommand: $(HelixPreCommandStemMusl);$(ExtraMSBuildLogsLinux) - - HelixPostCommand: 'export PYTHONPATH=$ORIGPYPATH' - - IsInternal: --internal - - ${{ if ne(parameters.osSubGroup, '_musl') }}: - - HelixPreCommand: $(HelixPreCommandStemLinux);$(ExtraMSBuildLogsLinux) - - HelixPostCommand: 'export PYTHONPATH=$ORIGPYPATH;${{ parameters.collectHelixLogsScript }}' - - IsInternal: --internal + - name: internalParam + value: --internal - group: DotNet-HelixApi-Access - group: dotnet-benchview - - ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}: - - ${{ if eq( parameters.osGroup, 'windows') }}: - - HelixPreCommand: $(HelixPreCommandStemWindows);$(ExtraMSBuildLogsWindows) - - HelixPostCommand: 'export PYTHONPATH=$ORIGPYPATH;${{ parameters.collectHelixLogsScript }}' - - ${{ if ne(parameters.osGroup, 'windows') }}: - - HelixPreCommand: $(HelixPreCommandStemLinux);$(ExtraMSBuildLogsLinux); - - HelixPostCommand: 'export PYTHONPATH=$ORIGPYPATH;${{ parameters.collectHelixLogsScript }}' - - - ${{ if and(eq(parameters.codeGenType, 'Interpreter'), eq(parameters.runtimeType, 'mono')) }}: - - ${{ if eq( parameters.osGroup, 'windows') }}: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - HelixPreCommand: '$(HelixPreCommandStemWindows);set MONO_ENV_OPTIONS="--interpreter";$(ExtraMSBuildLogsWindows)' - - Interpreter: ' -MonoInterpreter' - - ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}: - - HelixPreCommand: 'set MONO_ENV_OPTIONS="--interpreter";$(ExtraMSBuildLogsWindows)' - - Interpreter: ' -MonoInterpreter' - - ${{ if ne(parameters.osGroup, 'windows') }}: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - HelixPreCommand: '$(HelixPreCommandStemLinux);export MONO_ENV_OPTIONS="--interpreter";$(ExtraMSBuildLogsLinux)' - - Interpreter: ' --monointerpreter' - - ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}: - - HelixPreCommand: 'export MONO_ENV_OPTIONS="--interpreter";$(ExtraMSBuildLogsLinux)' - - Interpreter: ' --monointerpreter' - workspace: clean: all pool: @@ -172,28 +78,14 @@ jobs: _Framework: ${{ framework }} steps: - ${{ parameters.steps }} - - powershell: $(Build.SourcesDirectory)\eng\testing\performance\performance-setup.ps1 $(IsInternal)$(Interpreter) -Framework $(_Framework) -Kind ${{ parameters.runKind }} -LogicalMachine ${{ parameters.logicalMachine }} ${{ parameters.pgoRunType }} ${{ parameters.physicalPromotionRunType }} ${{ parameters.r2rRunType }} -ExperimentName '${{ parameters.experimentName }}' -UseLocalCommitTime ${{ parameters.extraSetupParameters }} - displayName: Performance Setup (Windows) - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $(Build.SourcesDirectory)/eng/testing/performance/performance-setup.sh $(IsInternal)$(Interpreter) --framework $(_Framework) --kind ${{ parameters.runKind }} --logicalmachine ${{ parameters.logicalMachine }} ${{ parameters.pgoRunType }} ${{ parameters.physicalPromotionRunType }} ${{ parameters.r2rRunType }} --experimentname '${{ parameters.experimentName }}' --uselocalcommittime ${{ parameters.extraSetupParameters }} - displayName: Performance Setup (Unix) - condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments) ${{ parameters.additionalSetupParameters }} - displayName: Run ci setup script + - script: $(Python) $(Agent.BuildDirectory)/performance/scripts/run_performance_job.py --framework $(_Framework) $(internalParam) ${{ parameters.extraSetupParameters }} ${{ parameters.additionalSetupParameters }} + displayName: Run performance job script + env: + HelixAccessToken: '$(HelixApiAccessToken)' + PerfCommandUploadToken: '$(PerfCommandUploadToken)' + PerfCommandUploadTokenLinux: '$(PerfCommandUploadTokenLinux)' # Run perf testing in helix - template: /eng/pipelines/coreclr/templates/perf-send-to-helix.yml parameters: - HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)${{ parameters.helixTypeSuffix }}' - HelixAccessToken: $(HelixApiAccessToken) - HelixTargetQueues: $(Queue) - HelixPreCommands: $(HelixPreCommand) - HelixPostCommands: $(HelixPostCommand) - Creator: $(Creator) - WorkItemTimeout: 4:00 # 4 hours - WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy - CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions ProjectFile: ${{ parameters.projectFile }} osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/coreclr/templates/run-scenarios-job.yml b/eng/pipelines/coreclr/templates/run-scenarios-job.yml index e58dd3586e7869..2bb05f14835773 100644 --- a/eng/pipelines/coreclr/templates/run-scenarios-job.yml +++ b/eng/pipelines/coreclr/templates/run-scenarios-job.yml @@ -48,62 +48,15 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} - - IsInternal: '' - - HelixApiAccessToken: '' - - SharedHelixPreCommands: '' - - AdditionalHelixPreCommands: '' - - AdditionalHelixPostCommands: '' - # run machine-setup and set PYTHONPATH for both public and private jobs - - ${{ if eq(parameters.osGroup, 'windows') }}: - - SharedHelixPreCommands: 'call %HELIX_WORKITEM_PAYLOAD%\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD%' - - ${{ if ne(parameters.osGroup, 'windows') }}: - - SharedHelixPreCommands: 'chmod +x $HELIX_WORKITEM_PAYLOAD/machine-setup.sh;. $HELIX_WORKITEM_PAYLOAD/machine-setup.sh;export PYTHONPATH=$HELIX_WORKITEM_PAYLOAD/scripts:$HELIX_WORKITEM_PAYLOAD' + - name: internalParam + value: '' - - ${{ if eq(parameters.osGroup, 'windows') }}: - - HelixPreCommandWindows: 'set ORIGPYPATH=%PYTHONPATH%;py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;echo on;set PYTHONPATH=;python -m pip install -U pip;python -m pip install azure.storage.blob==12.13.0;python -m pip install azure.storage.queue==12.4.0;python -m pip install urllib3==1.26.15;python -m pip install azure.identity==1.16.1;set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' - - HelixPostCommandsWindows: 'set PYTHONPATH=%ORIGPYPATH%' - - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.osGroup, 'osx'), ne(parameters.osSubGroup, '_musl')) }}: - - HelixPreCommandLinux: 'export ORIGPYPATH=$PYTHONPATH;export CRYPTOGRAPHY_ALLOW_OPENSSL_102=true;sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.13.0;pip3 install azure.storage.queue==12.4.0;pip3 install azure.identity==1.16.1;pip3 install urllib3==1.26.15;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' - - HelixPostCommandsLinux: 'export PYTHONPATH=$ORIGPYPATH' - - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.osGroup, 'osx'), eq(parameters.osSubGroup, '_musl')) }}: - - HelixPreCommandMusl: 'ulimit -n 4096;export ORIGPYPATH=$PYTHONPATH;sudo apk add py3-virtualenv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.13.0;pip3 install azure.storage.queue==12.4.0;pip3 install azure.identity==1.16.1;pip3 install urllib3==1.26.15;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' - - HelixPostCommandsMusl: 'export PYTHONPATH=$ORIGPYPATH' - - ${{ if eq(parameters.osGroup, 'osx') }}: - - HelixPreCommandOSX: 'export ORIGPYPATH=$PYTHONPATH;export CRYPTOGRAPHY_ALLOW_OPENSSL_102=true;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.13.0;pip3 install azure.storage.queue==12.4.0;pip3 install azure.identity==1.16.1;pip3 install urllib3==1.26.15;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' - - HelixPostCommandOSX: 'export PYTHONPATH=$ORIGPYPATH' - - # extra private job settings - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.osGroup, 'windows') }}: - - AdditionalHelixPreCommands: $(HelixPreCommandWindows) - - AdditionalHelixPostCommands: $(HelixPostCommandsWindows) - - IsInternal: -Internal - - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.osGroup, 'osx'), ne(parameters.osSubGroup, '_musl')) }}: - - AdditionalHelixPreCommands: $(HelixPreCommandLinux) - - AdditionalHelixPostCommands: $(HelixPostCommandsLinux) - - IsInternal: --internal - - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.osGroup, 'osx'), eq(parameters.osSubGroup, '_musl')) }}: - - AdditionalHelixPreCommands: $(HelixPreCommandMusl) - - AdditionalHelixPostCommands: $(HelixPostCommandsMusl) - - IsInternal: --internal - - ${{ if eq(parameters.osGroup, 'osx') }}: - - AdditionalHelixPreCommands: $(HelixPreCommandOSX) - - AdditionalHelixPostCommands: $(HelixPostCommandOSX) - - IsInternal: --internal + - name: internalParam + value: --internal - group: DotNet-HelixApi-Access - group: dotnet-benchview - - ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}: - - ${{ if eq(parameters.osGroup, 'windows') }}: - - AdditionalHelixPreCommands: $(HelixPreCommandWindows) - - AdditionalHelixPostCommands: $(HelixPostCommandsWindows) - - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.osGroup, 'osx')) }}: - - AdditionalHelixPreCommands: $(HelixPreCommandLinux) - - AdditionalHelixPostCommands: $(HelixPostCommandsLinux) - - ${{ if eq(parameters.osGroup, 'osx') }}: - - AdditionalHelixPreCommands: $(HelixPreCommandOSX) - - AdditionalHelixPostCommands: $(HelixPostCommandOSX) - - ExtraSetupArguments: '' - name: ExtraSetupArguments ${{ if ne(parameters.runtimeType, 'wasm') }}: @@ -125,23 +78,7 @@ jobs: _Framework: ${{ framework }} steps: - ${{ parameters.steps }} - # run performance-setup - - powershell: $(Build.SourcesDirectory)\eng\testing\performance\performance-setup.ps1 $(IsInternal) -Framework $(_Framework) -Kind ${{ parameters.runKind }} -LogicalMachine ${{ parameters.logicalMachine }} -UseLocalCommitTime ${{ parameters.extraSetupParameters }} ${{ parameters.additionalSetupParameters }} - displayName: Performance Setup (Windows) - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $(Build.SourcesDirectory)/eng/testing/performance/performance-setup.sh $(IsInternal) --framework $(_Framework) --kind ${{ parameters.runKind }} --logicalmachine ${{ parameters.logicalMachine }} --uselocalcommittime ${{ parameters.extraSetupParameters }} ${{ parameters.additionalSetupParameters }} - displayName: Performance Setup (Linux/MAC) - condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - # run ci-setup - - script: $(Python) $(PerformanceDirectory)\scripts\ci_setup.py $(SetupArguments) $(ExtraSetupArguments) --output-file $(WorkItemDirectory)\machine-setup.cmd - displayName: Run ci setup script (Windows) - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) - - script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments) $(ExtraSetupArguments) --output-file $(WorkItemDirectory)/machine-setup.sh - displayName: Run ci setup script (Linux/MAC) - condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) - # copy wasm packs if running on wasm + # copy wasm packs if running on wasm (TODO: move to run-performance-job.py) - script: >- mkdir -p $(librariesDownloadDir)/bin/wasm/data && cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-latest $(librariesDownloadDir)/bin/wasm && @@ -150,67 +87,16 @@ jobs: find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \; displayName: "Create wasm directory (Linux)" condition: and(succeeded(), eq('${{ parameters.runtimeType }}', 'wasm')) - # copy scenario support files - - script: xcopy $(PerformanceDirectory)\scripts $(WorkItemDirectory)\scripts\/e && xcopy $(PerformanceDirectory)\src\scenarios\shared $(WorkItemDirectory)\shared\/e && xcopy $(PerformanceDirectory)\src\scenarios\staticdeps $(WorkItemDirectory)\staticdeps\/e - displayName: Copy scenario support files (Windows) - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) - - script: cp -r $(PerformanceDirectory)/scripts $(WorkItemDirectory)/scripts/ && cp -r $(PerformanceDirectory)/src/scenarios/shared $(WorkItemDirectory)/shared/ && cp -r $(PerformanceDirectory)/src/scenarios/staticdeps/ $(WorkItemDirectory)/staticdeps/ - displayName: Copy scenario support files (Linux/MAC) - condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) - # build Startup - - script: $(PayloadDirectory)\dotnet\dotnet.exe publish -c Release -o $(WorkItemDirectory)\Startup -f net8.0 -r win-$(Architecture) --self-contained $(PerformanceDirectory)\src\tools\ScenarioMeasurement\Startup\Startup.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true - displayName: Build Startup tool (Windows) - env: - PERFLAB_TARGET_FRAMEWORKS: net8.0 - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) - - script: $(PayloadDirectory)/dotnet/dotnet publish -c Release -o $(WorkItemDirectory)/startup -f net8.0 -r linux-$(Architecture) --self-contained $(PerformanceDirectory)/src/tools/ScenarioMeasurement/Startup/Startup.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true - displayName: Build Startup tool (Linux) - env: - PERFLAB_TARGET_FRAMEWORKS: net8.0 - condition: and(succeeded(), eq(variables['Agent.Os'], 'Linux')) - - script: $(PayloadDirectory)/dotnet/dotnet publish -c Release -o $(WorkItemDirectory)/startup -f net8.0 -r osx-$(Architecture) --self-contained $(PerformanceDirectory)/src/tools/ScenarioMeasurement/Startup/Startup.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true - displayName: Build Startup tool (MAC) + # run run-performance-job.py + - script: $(Python) $(Agent.BuildDirectory)/performance/scripts/run_performance_job.py --is-scenario --framework $(_Framework) $(internalParam) ${{ parameters.extraSetupParameters }} ${{ parameters.additionalSetupParameters }} + displayName: Run performance job script env: - PERFLAB_TARGET_FRAMEWORKS: net8.0 - condition: and(succeeded(), eq(variables['Agent.Os'], 'Darwin')) - # build SizeOnDisk - - script: $(PayloadDirectory)\dotnet\dotnet.exe publish -c Release -o $(WorkItemDirectory)\SOD -f net8.0 -r win-$(Architecture) --self-contained $(PerformanceDirectory)\src\tools\ScenarioMeasurement\SizeOnDisk\SizeOnDisk.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true - displayName: Build SizeOnDisk tool (Windows) - env: - PERFLAB_TARGET_FRAMEWORKS: net8.0 - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) - - script: $(PayloadDirectory)/dotnet/dotnet publish -c Release -o $(WorkItemDirectory)/SOD -f net8.0 -r linux-$(Architecture) --self-contained $(PerformanceDirectory)/src/tools/ScenarioMeasurement/SizeOnDisk/SizeOnDisk.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true - displayName: Build SizeOnDisk tool (Linux) - env: - PERFLAB_TARGET_FRAMEWORKS: net8.0 - condition: and(succeeded(), eq(variables['Agent.Os'], 'Linux')) - - script: $(PayloadDirectory)/dotnet/dotnet publish -c Release -o $(WorkItemDirectory)/SOD -f net8.0 -r osx-$(Architecture) --self-contained $(PerformanceDirectory)/src/tools/ScenarioMeasurement/SizeOnDisk/SizeOnDisk.csproj -p:DisableTransitiveFrameworkReferenceDownloads=true - displayName: Build SizeOnDisk tool (MAC) - env: - PERFLAB_TARGET_FRAMEWORKS: net8.0 - condition: and(succeeded(), eq(variables['Agent.Os'], 'Darwin')) - - # Zip the workitem directory (for xharness (mobile) based workitems) - - ${{ if or(eq(parameters.runKind, 'android_scenarios'), eq(parameters.runKind, 'ios_scenarios')) }}: - - task: ArchiveFiles@2 - inputs: - rootFolderOrFile: '$(WorkItemDirectory)' - includeRootFolder: false - archiveFile: '$(WorkItemDirectory).zip' - verbose: True + HelixAccessToken: '$(HelixApiAccessToken)' + PerfCommandUploadToken: '$(PerfCommandUploadToken)' + PerfCommandUploadTokenLinux: '$(PerfCommandUploadTokenLinux)' # run perf testing in helix - template: /eng/pipelines/coreclr/templates/perf-send-to-helix.yml parameters: - HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)' - HelixAccessToken: $(HelixApiAccessToken) - HelixTargetQueues: $(Queue) - HelixPreCommands: '$(AdditionalHelixPreCommands);$(SharedHelixPreCommands)' # $(HelixPreCommands) should follow $(AdditionalHelixPreCommands) because PYTHONPATH is cleared by the former - HelixPostCommands: $(AdditionalHelixPostCommands) - Creator: $(Creator) - WorkItemTimeout: 4:00 # 4 hours - WorkItemDirectory: '$(WorkItemDirectory)' # contains scenario tools, shared python scripts, dotnet tool - CorrelationPayloadDirectory: '$(PayloadDirectory)' # contains performance repo and built product ProjectFile: ${{ parameters.projectFile }} osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/runtime-wasm-perf.yml b/eng/pipelines/runtime-wasm-perf.yml index 39645a501ecf25..b07b619773398c 100644 --- a/eng/pipelines/runtime-wasm-perf.yml +++ b/eng/pipelines/runtime-wasm-perf.yml @@ -2,6 +2,12 @@ # wasm jobs. This file is essentially so we can point the pipeline in azdo # UI to this, and thus avoid any scheduled triggers +parameters: +- name: perfBranch + displayName: Performance Repo Branch + type: string + default: 'main' + trigger: none pr: @@ -32,9 +38,7 @@ extends: runProfile: 'v8' collectHelixLogsScript: ${{ variables._wasmCollectHelixLogsScript }} onlySanityCheck: true - #perfForkToUse: # dummy change - #url: https://github.com/radical/performance - #branch: fix-build + perfBranch: ${{ parameters.perfBranch }} #downloadSpecificBuild: #buildId: '1878694' #pipeline: 'perf-wasm' diff --git a/eng/testing/performance/android_scenarios.proj b/eng/testing/performance/android_scenarios.proj index 4d0aad300cd99d..fb18ea62bbb027 100644 --- a/eng/testing/performance/android_scenarios.proj +++ b/eng/testing/performance/android_scenarios.proj @@ -16,40 +16,40 @@ - %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ + %HELIX_WORKITEM_ROOT%\performance\src\scenarios\ - $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ + $HELIX_WORKITEM_ROOT/performance/src/scenarios/ $(WorkItemDirectory) - cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroid.apk .;$(Python) pre.py --apk-name HelloAndroid.apk + cd $(ScenarioDirectory)helloandroid;copy %HELIX_WORKITEM_ROOT%\HelloAndroid.apk .;$(Python) pre.py --apk-name HelloAndroid.apk $(Python) test.py sod --scenario-name "%(Identity)" $(Python) post.py $(WorkItemDirectory) - cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroid.apk .;$(Python) pre.py --unzip --apk-name HelloAndroid.apk + cd $(ScenarioDirectory)helloandroid;copy %HELIX_WORKITEM_ROOT%\HelloAndroid.apk .;$(Python) pre.py --unzip --apk-name HelloAndroid.apk $(Python) test.py sod --scenario-name "%(Identity)" $(Python) post.py $(WorkItemDirectory) - cd $(ScenarioDirectory)bdnandroid;copy %HELIX_CORRELATION_PAYLOAD%\MonoBenchmarksDroid.apk .;$(Python) pre.py --apk-name MonoBenchmarksDroid.apk + cd $(ScenarioDirectory)bdnandroid;copy %HELIX_WORKITEM_ROOT%\MonoBenchmarksDroid.apk .;$(Python) pre.py --apk-name MonoBenchmarksDroid.apk $(Python) test.py sod --scenario-name "%(Identity)" $(Python) post.py $(WorkItemDirectory) - cd $(ScenarioDirectory)bdnandroid;copy %HELIX_CORRELATION_PAYLOAD%\MonoBenchmarksDroid.apk .;$(Python) pre.py --unzip --apk-name MonoBenchmarksDroid.apk + cd $(ScenarioDirectory)bdnandroid;copy %HELIX_WORKITEM_ROOT%\MonoBenchmarksDroid.apk .;$(Python) pre.py --unzip --apk-name MonoBenchmarksDroid.apk $(Python) test.py sod --scenario-name "%(Identity)" $(Python) post.py $(WorkItemDirectory) - echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)bdnandroid;copy %HELIX_CORRELATION_PAYLOAD%\MonoBenchmarksDroid.apk .;$(Python) pre.py --restart-device --apk-name MonoBenchmarksDroid.apk + echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)bdnandroid;copy %HELIX_WORKITEM_ROOT%\MonoBenchmarksDroid.apk .;$(Python) pre.py --restart-device --apk-name MonoBenchmarksDroid.apk $(Python) test.py androidinstrumentation --package-path .\pub\MonoBenchmarksDroid.apk --package-name com.microsoft.maui.benchmarks --instrumentation-name com.microsoft.maui.MainInstrumentation --scenario-name "%(Identity)" $(Python) post.py 00:30:00 diff --git a/eng/testing/performance/blazor_perf.proj b/eng/testing/performance/blazor_perf.proj index bacb52650c140f..45ad650dc1d3c9 100644 --- a/eng/testing/performance/blazor_perf.proj +++ b/eng/testing/performance/blazor_perf.proj @@ -20,7 +20,7 @@ - %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ + %HELIX_WORKITEM_ROOT%\performance\src\scenarios\ $(ScenarioDirectory)blazorminapp\ $(ScenarioDirectory)blazor\ $(ScenarioDirectory)blazorpizza\ @@ -32,7 +32,7 @@ pub\wwwroot - $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ + $HELIX_WORKITEM_ROOT/performance/src/scenarios/ $(ScenarioDirectory)blazorminapp/ $(ScenarioDirectory)blazor/ $(ScenarioDirectory)blazorpizza/ diff --git a/eng/testing/performance/crossgen_perf.proj b/eng/testing/performance/crossgen_perf.proj index 252ec7eaedb198..4b7d163eaa238c 100644 --- a/eng/testing/performance/crossgen_perf.proj +++ b/eng/testing/performance/crossgen_perf.proj @@ -13,7 +13,7 @@ python $(HelixPreCommands) %HELIX_CORRELATION_PAYLOAD%\Core_Root - %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ + %HELIX_WORKITEM_ROOT%\performance\src\scenarios\ $(ScenarioDirectory)crossgen\ $(ScenarioDirectory)crossgen2\ @@ -21,7 +21,7 @@ python3 $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/startup/Startup;chmod +x $HELIX_WORKITEM_PAYLOAD/startup/perfcollect;sudo apt update;chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk $HELIX_CORRELATION_PAYLOAD/Core_Root - $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ + $HELIX_WORKITEM_ROOT/performance/src/scenarios/ $(ScenarioDirectory)crossgen/ $(ScenarioDirectory)crossgen2/ diff --git a/eng/testing/performance/ios_scenarios.proj b/eng/testing/performance/ios_scenarios.proj index 1e4c59acd2ba3b..1bc2a4ef5f926f 100644 --- a/eng/testing/performance/ios_scenarios.proj +++ b/eng/testing/performance/ios_scenarios.proj @@ -25,11 +25,11 @@ - %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ + %HELIX_WORKITEM_ROOT%\performance\src\scenarios\ - $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ + $HELIX_WORKITEM_ROOT/performance/src/scenarios/ diff --git a/eng/testing/performance/microbenchmarks.proj b/eng/testing/performance/microbenchmarks.proj deleted file mode 100644 index 8453f469cd0af6..00000000000000 --- a/eng/testing/performance/microbenchmarks.proj +++ /dev/null @@ -1,170 +0,0 @@ - - - - %HELIX_WORKITEM_ROOT%\performance - $(HelixPreCommands) && robocopy /np /nfl /e %HELIX_CORRELATION_PAYLOAD%\performance $(PerformanceDirectory) /XD %HELIX_CORRELATION_PAYLOAD%\performance\.git - $(PerformanceDirectory)\scripts\benchmarks_ci.py --csproj $(PerformanceDirectory)\$(TargetCsproj) - --dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP% - python - %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe - %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe - $(HelixPreCommands);call $(PerformanceDirectory)\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% - %HELIX_WORKITEM_ROOT%\artifacts\BenchmarkDotNet.Artifacts - %HELIX_WORKITEM_ROOT%\artifacts\BenchmarkDotNet.Artifacts_Baseline - $(PerformanceDirectory)\src\tools\ResultsComparer\ResultsComparer.csproj - $(PerformanceDirectory)\tools\dotnet\$(Architecture)\dotnet.exe - %25%25 - %HELIX_WORKITEM_ROOT%\testResults.xml - - - - $HELIX_CORRELATION_PAYLOAD - $HELIX_WORKITEM_ROOT/performance - $(HelixPreCommands);cp -R $(BaseDirectory)/performance $(PerformanceDirectory) - - - - $HELIX_WORKITEM_PAYLOAD - $(BaseDirectory) - - - - $(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj) - --dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP - python3 - $(BaseDirectory)/Core_Root/corerun - $(BaseDirectory)/Baseline_Core_Root/corerun - $(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh - $HELIX_WORKITEM_ROOT/artifacts/BenchmarkDotNet.Artifacts - $HELIX_WORKITEM_ROOT/artifacts/BenchmarkDotNet.Artifacts_Baseline - $(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj - $(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet - %25 - $HELIX_WORKITEM_ROOT/testResults.xml - - - - $(CliArguments) --run-isolated --wasm --dotnet-path %24HELIX_CORRELATION_PAYLOAD/dotnet/ - - - - --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\$(ProductVersion)\corerun.exe - - - --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/$(ProductVersion)/corerun - - - - $(HelixPreCommands);%HELIX_CORRELATION_PAYLOAD%\monoaot\mono-aot-cross --llvm --version - - - $(HelixPreCommands);$HELIX_CORRELATION_PAYLOAD/monoaot/mono-aot-cross --llvm --version - - - - --corerun $(CoreRun) - - - - --corerun $(BaselineCoreRun) - - - - $(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(PERFLAB_Framework) $(PerfLabArguments) - - - - $(WorkItemCommand) $(CliArguments) - - - - 6:00 - 1:30 - - $(ExtraBenchmarkDotNetArguments) --filter System.Tests.Perf_* - - - - - %(Identity) - - - - - 30 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - false - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - - if [ "x$PERF_PREREQS_INSTALL_FAILED" = "x1" ]; then - echo "\n\n** Error: Failed to install prerequisites **\n\n"; (exit 1); - else - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"; - fi - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(DotnetExe) run -f $(PERFLAB_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) - $(WorkItemTimeout) - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)" - - if [ "x$PERF_PREREQS_INSTALL_FAILED" = "x1" ]; then - echo "\n\n** Error: Failed to install prerequisites **\n\n"; (exit 1); - else - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)"; - fi - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)" - $(DotnetExe) run -f $(PERFLAB_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults) - 4:00 - - - diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 deleted file mode 100644 index 8e9ff87364699b..00000000000000 --- a/eng/testing/performance/performance-setup.ps1 +++ /dev/null @@ -1,228 +0,0 @@ -Param( - [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, - [string] $CoreRootDirectory, - [string] $BaselineCoreRootDirectory, - [string] $Architecture="x64", - [string] $Framework, - [string] $CompilationMode="Tiered", - [string] $Repository=$env:BUILD_REPOSITORY_NAME, - [string] $Branch=$env:BUILD_SOURCEBRANCH, - [string] $CommitSha=$env:BUILD_SOURCEVERSION, - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, - [string] $RunCategories="Libraries Runtime", - [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", - [string] $Kind="micro", - [switch] $LLVM, - [switch] $MonoInterpreter, - [switch] $MonoAOT, - [switch] $Internal, - [switch] $Compare, - [string] $MonoDotnet="", - [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind", - [string] $LogicalMachine="", - [switch] $AndroidMono, - [switch] $iOSMono, - [switch] $iOSNativeAOT, - [switch] $NoDynamicPGO, - [switch] $PhysicalPromotion, - [switch] $NoR2R, - [string] $ExperimentName, - [switch] $iOSLlvmBuild, - [switch] $iOSStripSymbols, - [switch] $HybridGlobalization, - [string] $MauiVersion, - [switch] $UseLocalCommitTime -) - -$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") -$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty) -$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty) - -$PayloadDirectory = (Join-Path $SourceDirectory "Payload") -$PerformanceDirectory = (Join-Path $PayloadDirectory "performance") -$WorkItemDirectory = (Join-Path $SourceDirectory "workitem") -$ExtraBenchmarkDotNetArguments = "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" -$Creator = $env:BUILD_DEFINITIONNAME -$PerfLabArguments = "" -$HelixSourcePrefix = "pr" - -$Queue = "" - -if ($Internal) { - switch ($LogicalMachine) { - "perftiger" { $Queue = "Windows.11.Amd64.Tiger.Perf" } - "perftiger_crossgen" { $Queue = "Windows.11.Amd64.Tiger.Perf" } - "perfowl" { $Queue = "Windows.11.Amd64.Owl.Perf" } - "perfsurf" { $Queue = "Windows.11.Arm64.Surf.Perf" } - "perfpixel4a" { $Queue = "Windows.11.Amd64.Pixel.Perf" } - "perfampere" { $Queue = "Windows.Server.Arm64.Perf" } - "perfviper" { $Queue = "Windows.11.Amd64.Viper.Perf" } - "cloudvm" { $Queue = "Windows.10.Amd64" } - Default { $Queue = "Windows.11.Amd64.Tiger.Perf" } - } - $PerfLabArguments = "--upload-to-perflab-container" - $ExtraBenchmarkDotNetArguments = "" - $Creator = "" - $HelixSourcePrefix = "official" -} -else { - $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" -} - -if ($MonoInterpreter) { - $ExtraBenchmarkDotNetArguments = "--category-exclusion-filter NoInterpreter" -} - -if ($MonoDotnet -ne "") { - $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" - if($ExtraBenchmarkDotNetArguments -eq "") - { - #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" - } - else - { - #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" - } -} - -if ($NoDynamicPGO) { - $Configurations += " PGOType=nodynamicpgo" -} - -if ($PhysicalPromotion) { - $Configurations += " PhysicalPromotionType=physicalpromotion" -} - -if ($NoR2R) { - $Configurations += " R2RType=nor2r" -} - -if ($ExperimentName) { - $Configurations += " ExperimentName=$ExperimentName" - if ($ExperimentName -eq "memoryRandomization") { - $ExtraBenchmarkDotNetArguments += " --memoryRandomization true" - } -} - -if ($iOSMono) { - $Configurations += " iOSLlvmBuild=$iOSLlvmBuild" - $Configurations += " iOSStripSymbols=$iOSStripSymbols" -} - -if ($iOSNativeAOT) { - $Configurations += " iOSStripSymbols=$iOSStripSymbols" -} - -if ($HybridGlobalization -eq "True") { - $Configurations += " HybridGlobalization=True" -} - -# FIX ME: This is a workaround until we get this from the actual pipeline -$CleanedBranchName = "main" -if($Branch.Contains("refs/heads/release")) -{ - $CleanedBranchName = $Branch.replace('refs/heads/', '') -} -$CommonSetupArguments="--channel $CleanedBranchName --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" -$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" - -if ($NoDynamicPGO) { - $SetupArguments = "$SetupArguments --no-dynamic-pgo" -} - -if ($PhysicalPromotion) { - $SetupArguments = "$SetupArguments --physical-promotion" -} - -if ($NoR2R) { - $SetupArguments = "$SetupArguments --no-r2r" -} - -if ($ExperimentName) { - $SetupArguments = "$SetupArguments --experiment-name $ExperimentName" -} - -if ($UseLocalCommitTime) { - $LocalCommitTime = (git show -s --format=%ci $CommitSha) - $SetupArguments = "$SetupArguments --commit-time `"$LocalCommitTime`"" -} - -if ($RunFromPerformanceRepo) { - $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" - - robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git -} -else { - git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory -} - -if ($MonoDotnet -ne "") { - $UsingMono = "true" - $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") - Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath -} - -if ($UseCoreRun) { - $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") - Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot -} -if ($UseBaselineCoreRun) { - $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root") - Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot -} - -if ($MauiVersion -ne "") { - $SetupArguments = "$SetupArguments --maui-version $MauiVersion" -} - -if ($AndroidMono) { - if(!(Test-Path $WorkItemDirectory)) - { - mkdir $WorkItemDirectory - } - Copy-Item -path "$SourceDirectory\MonoBenchmarksDroid.apk" $PayloadDirectory -Verbose - Copy-Item -path "$SourceDirectory\androidHelloWorld\HelloAndroid.apk" $PayloadDirectory -Verbose - $SetupArguments = $SetupArguments -replace $Architecture, 'arm64' -} - -$DocsDir = (Join-Path $PerformanceDirectory "docs") -robocopy $DocsDir $WorkItemDirectory - -# Set variables that we will need to have in future steps -$ci = $true - -. "$PSScriptRoot\..\..\common\pipeline-logging-functions.ps1" - -# Directories -Write-PipelineSetVariable -Name 'PayloadDirectory' -Value "$PayloadDirectory" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'PerformanceDirectory' -Value "$PerformanceDirectory" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'WorkItemDirectory' -Value "$WorkItemDirectory" -IsMultiJobVariable $false - -# Script Arguments -Write-PipelineSetVariable -Name 'Python' -Value "py -3" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'ExtraBenchmarkDotNetArguments' -Value "$ExtraBenchmarkDotNetArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'SetupArguments' -Value "$SetupArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'PerfLabArguments' -Value "$PerfLabArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'BDNCategories' -Value "$RunCategories" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'MonoAOT' -Value "$MonoAOT" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'iOSLlvmBuild' -Value "$iOSLlvmBuild" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'iOSStripSymbols' -Value "$iOSStripSymbols" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'hybridGlobalization' -Value "$HybridGlobalization" -IsMultiJobVariable $false - -# Helix Arguments -Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Queue' -Value "$Queue" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'HelixSourcePrefix' -Value "$HelixSourcePrefix" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name '_BuildConfig' -Value "$Architecture.$Kind.$Framework" -IsMultiJobVariable $false - -exit 0 diff --git a/eng/testing/performance/performance-setup.sh b/eng/testing/performance/performance-setup.sh deleted file mode 100755 index 489b3a22cccd5f..00000000000000 --- a/eng/testing/performance/performance-setup.sh +++ /dev/null @@ -1,564 +0,0 @@ -#!/usr/bin/env bash - -# Also reset/set below -set -x - -source_directory=$BUILD_SOURCESDIRECTORY -core_root_directory= -baseline_core_root_directory= -architecture=x64 -framework= -compilation_mode=tiered -repository=$BUILD_REPOSITORY_NAME -branch=$BUILD_SOURCEBRANCH -commit_sha=$BUILD_SOURCEVERSION -build_number=$BUILD_BUILDNUMBER -internal=false -compare=false -mono_dotnet= -kind="micro" -llvm=false -monointerpreter=false -monoaot=false -monoaot_path= -run_categories="Libraries Runtime" -csproj="src\benchmarks\micro\MicroBenchmarks.csproj" -configurations="CompilationMode=$compilation_mode RunKind=$kind" -perf_fork="" -perf_fork_branch="main" -run_from_perf_repo=false -use_core_run=true -use_baseline_core_run=true -using_mono=false -wasm_bundle_directory= -using_wasm=false -use_latest_dotnet=false -logical_machine= -javascript_engine="v8" -javascript_engine_path="" -iosmono=false -iosnativeaot=false -runtimetype="" -iosllvmbuild="" -iosstripsymbols="" -hybridglobalization="" -maui_version="" -use_local_commit_time=false -only_sanity=false -dotnet_versions="" -v8_version="" - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --sourcedirectory) - source_directory=$2 - shift 2 - ;; - --corerootdirectory) - core_root_directory=$2 - shift 2 - ;; - --baselinecorerootdirectory) - baseline_core_root_directory=$2 - shift 2 - ;; - --architecture) - architecture=$2 - shift 2 - ;; - --framework) - framework=$2 - shift 2 - ;; - --compilationmode) - compilation_mode=$2 - shift 2 - ;; - --logicalmachine) - logical_machine=$2 - shift 2 - ;; - --repository) - repository=$2 - shift 2 - ;; - --branch) - branch=$2 - shift 2 - ;; - --commitsha) - commit_sha=$2 - shift 2 - ;; - --buildnumber) - build_number=$2 - shift 2 - ;; - --javascriptengine) - javascript_engine=$2 - shift 2 - ;; - --javascriptenginepath) - javascript_engine_path=$2 - shift 2 - ;; - --kind) - kind=$2 - configurations="CompilationMode=$compilation_mode RunKind=$kind" - shift 2 - ;; - --runcategories) - run_categories=$2 - shift 2 - ;; - --csproj) - csproj=$2 - shift 2 - ;; - --internal) - internal=true - shift 1 - ;; - --alpine) - alpine=true - shift 1 - ;; - --llvm) - llvm=true - shift 1 - ;; - --monointerpreter) - monointerpreter=true - shift 1 - ;; - --monoaot) - monoaot=true - monoaot_path=$2 - shift 2 - ;; - --monodotnet) - mono_dotnet=$2 - shift 2 - ;; - --wasmbundle) - wasm_bundle_directory=$2 - shift 2 - ;; - --wasmaot) - wasmaot=true - shift 1 - ;; - --nodynamicpgo) - nodynamicpgo=true - shift 1 - ;; - --physicalpromotion) - physicalpromotion=true - shift 1 - ;; - --nor2r) - nor2r=true - shift 1 - ;; - --experimentname) - experimentname=$2 - shift 2 - ;; - --compare) - compare=true - shift 1 - ;; - --configurations) - configurations=$2 - shift 2 - ;; - --latestdotnet) - use_latest_dotnet=true - shift 1 - ;; - --dotnetversions) - dotnet_versions="$2" - shift 2 - ;; - --iosmono) - iosmono=true - shift 1 - ;; - --iosnativeaot) - iosnativeaot=true - shift 1 - ;; - --iosllvmbuild) - iosllvmbuild=$2 - shift 2 - ;; - --iosstripsymbols) - iosstripsymbols=$2 - shift 2 - ;; - --hybridglobalization) - hybridglobalization=$2 - shift 2 - ;; - --mauiversion) - maui_version=$2 - shift 2 - ;; - --uselocalcommittime) - use_local_commit_time=true - shift 1 - ;; - --perffork) - perf_fork=$2 - shift 2 - ;; - --perfforkbranch) - perf_fork_branch=$2 - shift 2 - ;; - --only-sanity) - only_sanity=true - shift 1 - ;; - *) - echo "Common settings:" - echo " --corerootdirectory Directory where Core_Root exists, if running perf testing with --corerun" - echo " --architecture Architecture of the testing being run" - echo " --configurations List of key=value pairs that will be passed to perf testing infrastructure." - echo " ex: --configurations \"CompilationMode=Tiered OptimzationLevel=PGO\"" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --framework The framework to run, if not running in master" - echo " --compilationmode The compilation mode if not passing --configurations" - echo " --sourcedirectory The directory of the sources. Defaults to env:BUILD_SOURCESDIRECTORY" - echo " --repository The name of the repository in the / format. Defaults to env:BUILD_REPOSITORY_NAME" - echo " --branch The name of the branch. Defaults to env:BUILD_SOURCEBRANCH" - echo " --commitsha The commit sha1 to run against. Defaults to env:BUILD_SOURCEVERSION" - echo " --buildnumber The build number currently running. Defaults to env:BUILD_BUILDNUMBER" - echo " --csproj The relative path to the benchmark csproj whose tests should be run. Defaults to src\benchmarks\micro\MicroBenchmarks.csproj" - echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" - echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" - echo " --internal If the benchmarks are running as an official job." - echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." - echo " --wasmbundle Path to the wasm bundle containing the dotnet, and data needed for helix payload" - echo " --wasmaot Indicate wasm aot" - echo " --latestdotnet --dotnet-versions will not be specified. --dotnet-versions defaults to LKG version in global.json " - echo " --dotnetversions Passed as '--dotnet-versions ' to the setup script" - echo " --alpine Set for runs on Alpine" - echo " --llvm Set LLVM for Mono runs" - echo " --iosmono Set for ios Mono/Maui runs" - echo " --iosnativeaot Set for ios Native AOT runs" - echo " --iosllvmbuild Set LLVM for iOS Mono/Maui runs" - echo " --iosstripsymbols Set STRIP_DEBUG_SYMBOLS for iOS Mono/Maui runs" - echo " --hybridglobalization Set hybrid globalization for iOS Mono/Maui/Wasm runs" - echo " --mauiversion Set the maui version for Mono/Maui runs" - echo " --uselocalcommittime Pass local runtime commit time to the setup script" - echo " --nodynamicpgo Set for No dynamic PGO runs" - echo " --physicalpromotion Set for runs with physical promotion" - echo " --nor2r Set for No R2R runs" - echo " --experimentname Set Experiment Name" - echo "" - exit 1 - ;; - esac -done - -if [[ "$repository" == "dotnet/performance" || "$repository" == "dotnet-performance" ]]; then - run_from_perf_repo=true -fi - -if [ -z "$configurations" ]; then - configurations="CompilationMode=$compilation_mode" -fi - -if [ -z "$core_root_directory" ]; then - use_core_run=false -fi - -if [ -z "$baseline_core_root_directory" ]; then - use_baseline_core_run=false -fi - -payload_directory=$source_directory/Payload -performance_directory=$payload_directory/performance -benchmark_directory=$payload_directory/BenchmarkDotNet -workitem_directory=$source_directory/workitem -extra_benchmark_dotnet_arguments="--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" -perflab_arguments= -queue=Ubuntu.2204.Amd64.Open -creator=$BUILD_DEFINITIONNAME -helix_source_prefix="pr" - -if [[ "$internal" == true ]]; then - perflab_arguments="--upload-to-perflab-container" - helix_source_prefix="official" - creator= - extra_benchmark_dotnet_arguments= - - if [[ "$logical_machine" == "perfiphone12mini" ]]; then - queue=OSX.13.Amd64.Iphone.Perf - elif [[ "$logical_machine" == "perfampere" ]]; then - queue=Ubuntu.2204.Arm64.Perf - elif [[ "$logical_machine" == "perfviper" ]]; then - queue=Ubuntu.2204.Amd64.Viper.Perf - elif [[ "$logical_machine" == "cloudvm" ]]; then - queue=Ubuntu.2204.Amd64 - elif [[ "$architecture" == "arm64" ]]; then - queue=Ubuntu.1804.Arm64.Perf - else - if [[ "$logical_machine" == "perfowl" ]]; then - queue=Ubuntu.2204.Amd64.Owl.Perf - elif [[ "$logical_machine" == "perftiger_crossgen" ]]; then - queue=Ubuntu.1804.Amd64.Tiger.Perf - else - queue=Ubuntu.2204.Amd64.Tiger.Perf - fi - fi - - if [[ "$alpine" == "true" ]]; then - queue=alpine.amd64.tiger.perf - fi -else - if [[ "$architecture" == "arm64" ]]; then - queue=ubuntu.1804.armarch.open - else - queue=Ubuntu.2204.Amd64.Open - fi - - if [[ "$alpine" == "true" ]]; then - queue=alpine.amd64.tiger.perf - fi -fi - -if [[ -n "$mono_dotnet" && "$monointerpreter" == "false" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono" -fi - -_BuildConfig="$architecture.$kind.$framework" - -if [[ -n "$wasm_bundle_directory" ]]; then - if [[ "$wasmaot" == "true" ]]; then - configurations="CompilationMode=wasm AOT=true RunKind=$kind" - _BuildConfig="wasmaot.$_BuildConfig" - else - configurations="CompilationMode=wasm RunKind=$kind" - _BuildConfig="wasm.$_BuildConfig" - fi - if [[ "$javascript_engine" == "javascriptcore" ]]; then - configurations="$configurations JSEngine=javascriptcore" - fi - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM NoMono" -fi - -if [[ -n "$mono_dotnet" && "$monointerpreter" == "true" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono" -fi - -if [[ "$monoaot" == "true" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoAOT NoWASM" -fi - -if [[ "$iosmono" == "true" ]]; then - runtimetype="Mono" - configurations="$configurations iOSLlvmBuild=$iosllvmbuild iOSStripSymbols=$iosstripsymbols RuntimeType=$runtimetype" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments" -fi - -if [[ "$iosnativeaot" == "true" ]]; then - runtimetype="NativeAOT" - configurations="$configurations iOSStripSymbols=$iosstripsymbols RuntimeType=$runtimetype" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments" -fi - -if [[ "$nodynamicpgo" == "true" ]]; then - configurations="$configurations PGOType=nodynamicpgo" -fi - -if [[ "$physicalpromotion" == "true" ]]; then - configurations="$configurations PhysicalPromotionType=physicalpromotion" -fi - -if [[ "$nor2r" == "true" ]]; then - configurations="$configurations R2RType=nor2r" -fi - -if [[ ! -z "$experimentname" ]]; then - configurations="$configurations ExperimentName=$experimentname" - if [[ "$experimentname" == "memoryRandomization" ]]; then - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --memoryRandomization true" - fi -fi - -if [[ "$(echo "$hybridglobalization" | tr '[:upper:]' '[:lower:]')" == "true" ]]; then # convert to lowercase to test - configurations="$configurations HybridGlobalization=True" # Force True for consistency -fi - - - -cleaned_branch_name="main" -if [[ $branch == *"refs/heads/release"* ]]; then - cleaned_branch_name=${branch/refs\/heads\//} -fi -common_setup_arguments="--channel $cleaned_branch_name --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" -setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" - -if [[ "$internal" != true ]]; then - setup_arguments="$setup_arguments --not-in-lab" -fi - -if [[ "$use_local_commit_time" == true ]]; then - local_commit_time=$(git show -s --format=%ci $commit_sha) - setup_arguments="$setup_arguments --commit-time \"$local_commit_time\"" -fi - -if [[ "$run_from_perf_repo" == true ]]; then - payload_directory= - workitem_directory=$source_directory - performance_directory=$workitem_directory - setup_arguments="--perf-hash $commit_sha $common_setup_arguments" -else - if [[ -n "$perf_fork" ]]; then - git clone --branch $perf_fork_branch --depth 1 --quiet $perf_fork $performance_directory - else - git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance.git $performance_directory - fi - # uncomment to use BenchmarkDotNet sources instead of nuget packages - # git clone https://github.com/dotnet/BenchmarkDotNet.git $benchmark_directory - - (cd $performance_directory; git show -s HEAD) - - docs_directory=$performance_directory/docs - mv $docs_directory $workitem_directory -fi - -if [[ -n "$maui_version" ]]; then - setup_arguments="$setup_arguments --maui-version $maui_version" -fi - -if [[ -n "$wasm_bundle_directory" ]]; then - using_wasm=true - wasm_bundle_directory_path=$payload_directory - mv $wasm_bundle_directory/* $wasm_bundle_directory_path - wasm_args="--expose_wasm" - if [ "$javascript_engine" == "v8" ]; then - # for es6 module support - wasm_args="$wasm_args --module" - - # get required version - if [[ -z "$v8_version" ]]; then - v8_version=`grep linux_V8Version $source_directory/eng/testing/BrowserVersions.props | sed -e 's,.*>\([^\<]*\)<.*,\1,g' | cut -d. -f 1-3` - echo "V8 version: $v8_version" - fi - if [[ -z "$javascript_engine_path" ]]; then - javascript_engine_path="/home/helixbot/.jsvu/bin/v8-${v8_version}" - fi - fi - - if [[ -z "$javascript_engine_path" ]]; then - javascript_engine_path="/home/helixbot/.jsvu/bin/$javascript_engine" - fi - - # Workaround: escaping the quotes around `--wasmArgs=..` so they get retained for the actual command line - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmEngine $javascript_engine_path \\\"--wasmArgs=$wasm_args\\\" --cli \$HELIX_CORRELATION_PAYLOAD/dotnet/dotnet --wasmDataDir \$HELIX_CORRELATION_PAYLOAD/wasm-data" - if [[ "$wasmaot" == "true" ]]; then - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --aotcompilermode wasm --buildTimeout 3600" - fi - setup_arguments="$setup_arguments --dotnet-path $wasm_bundle_directory_path/dotnet" -fi - -if [[ -n "$mono_dotnet" && "$monoaot" == "false" ]]; then - using_mono=true - mono_dotnet_path=$payload_directory/dotnet-mono - mv $mono_dotnet $mono_dotnet_path -fi - -if [[ -n "$dotnet_versions" ]]; then - setup_arguments="$setup_arguments --dotnet-versions $dotnet_versions" -fi - -if [[ "$nodynamicpgo" == "true" ]]; then - setup_arguments="$setup_arguments --no-dynamic-pgo" -fi - -if [[ "$physicalpromotion" == "true" ]]; then - setup_arguments="$setup_arguments --physical-promotion" -fi - -if [[ "$nor2r" == "true" ]]; then - setup_arguments="$setup_arguments --no-r2r" -fi - -if [[ ! -z "$experimentname" ]]; then - setup_arguments="$setup_arguments --experiment-name $experimentname" -fi - -if [[ "$monoaot" == "true" ]]; then - monoaot_dotnet_path=$payload_directory/monoaot - mv $monoaot_path $monoaot_dotnet_path - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --runtimes monoaotllvm --aotcompilerpath \$HELIX_CORRELATION_PAYLOAD/monoaot/mono-aot-cross --customruntimepack \$HELIX_CORRELATION_PAYLOAD/monoaot/pack --aotcompilermode llvm" -fi - -extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --logBuildOutput --generateBinLog" - -if [[ "$use_core_run" == true ]]; then - new_core_root=$payload_directory/Core_Root - mv $core_root_directory $new_core_root -fi - -if [[ "$use_baseline_core_run" == true ]]; then - new_baseline_core_root=$payload_directory/Baseline_Core_Root - mv $baseline_core_root_directory $new_baseline_core_root -fi - -if [[ "$iosmono" == "true" || "$iosnativeaot" == "true" ]]; then - mkdir -p $payload_directory/iosHelloWorld && cp -rv $source_directory/iosHelloWorld $payload_directory/iosHelloWorld - mkdir -p $payload_directory/iosHelloWorldZip && cp -rv $source_directory/iosHelloWorldZip $payload_directory/iosHelloWorldZip - - find "$payload_directory/iosHelloWorldZip/" -type f -name "*.zip" -execdir mv {} "$payload_directory/iosHelloWorldZip/iOSSampleApp.zip" \; -fi - -ci=true - -_script_dir=$(pwd)/eng/common -. "$_script_dir/pipeline-logging-functions.sh" - -# Prevent vso[task.setvariable to be erroneously processed -set +x - -# Make sure all of our variables are available for future steps -Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false -Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false -Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false -Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false -Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "Python" -value "python3" -is_multi_job_variable false -Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false -Write-PipelineSetVariable -name "TargetCsproj" -value "$csproj" -is_multi_job_variable false -Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -is_multi_job_variable false -Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false -Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false -Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false -Write-PipelineSetVariable -name "_BuildConfig" -value "$_BuildConfig" -is_multi_job_variable false -Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false -Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false -Write-PipelineSetVariable -name "MonoAOT" -value "$monoaot" -is_multi_job_variable false -Write-PipelineSetVariable -name "WasmDotnet" -value "$using_wasm" -is_multi_job_variable false -Write-PipelineSetVariable -Name 'iOSLlvmBuild' -Value "$iosllvmbuild" -is_multi_job_variable false -Write-PipelineSetVariable -Name 'iOSStripSymbols' -Value "$iosstripsymbols" -is_multi_job_variable false -Write-PipelineSetVariable -Name 'hybridGlobalization' -Value "$hybridglobalization" -is_multi_job_variable false -Write-PipelineSetVariable -Name 'RuntimeType' -Value "$runtimetype" -is_multi_job_variable false -Write-PipelineSetVariable -name "OnlySanityCheck" -value "$only_sanity" -is_multi_job_variable false -Write-PipelineSetVariable -name "V8Version" -value "$v8_version" -is_multi_job_variable false - -# Put it back to what was set on top of this script -set -x