Use wasm-tools workload to run benchmarks for wasm (#66493)

* [wasm] perf jobs: Use workloads to build benchmark projects

This uses the new `--wasmDataDir` parameter. And `--cli` to use the
dotnet from `dotnet-workload` which has the `wasm-tools` workload
installed from artifacts.

* perf pipeline: Use a dotnet with workload installed from artifacts

.. for running the benchmarks.

* [wasm] perf pipeline: Keep only the needed bits in the BrowserWasm

.. download, instead of copying the whole `artifacts` folder.

* Skip downloading unncessary liveLibraries build

* [wasm] Skip unncessary core_root, and corelcr download

eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml

@@ -0,0 +1,27 @@
+parameters:
+ configForBuild: 'Release'
+
+steps:
+ - script: >-
+ ./dotnet.sh build -p:TargetOS=Browser -p:TargetArchitecture=wasm /nr:false /p:TreatWarningsAsErrors=true
+ /p:Configuration=${{ parameters.configForBuild }}
+ /p:ContinuousIntegrationBuild=true
+ /t:InstallWorkloadUsingArtifacts
+ $(Build.SourcesDirectory)/src/tests/BuildWasmApps/Wasm.Build.Tests/Wasm.Build.Tests.csproj
+ displayName: "Install workload using artifacts"
+
+ - script: >-
+ mkdir -p $(Build.SourcesDirectory)/artifacts/staging &&
+ cp -r $(Build.SourcesDirectory)/artifacts/bin/dotnet-workload $(Build.SourcesDirectory)/artifacts/staging &&
+ cp -r $(Build.SourcesDirectory)/artifacts/bin/microsoft.netcore.app.runtime.browser-wasm $(Build.SourcesDirectory)/artifacts/staging &&
+ cp -r $(Build.SourcesDirectory)/artifacts/bin/microsoft.netcore.app.ref $(Build.SourcesDirectory)/artifacts/staging
+ displayName: "Prepare artifacts staging directory"
+
+ - template: /eng/pipelines/common/upload-artifact-step.yml
+ parameters:
+ rootFolder: '$(Build.SourcesDirectory)/artifacts/staging'
+ includeRootFolder: true
+ displayName: Browser Wasm Artifacts
+ artifactName: BrowserWasm
+ archiveType: zip
+ archiveExtension: .zip

eng/pipelines/coreclr/perf.yml

@@ -57,28 +57,23 @@ jobs:
  - template: /eng/pipelines/common/platform-matrix.yml
  parameters:
  jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: release
+ buildConfig: Release
  runtimeFlavor: mono
  platforms:
  - Browser_wasm
  jobParameters:
  buildArgs: -s mono+libs+host+packs -c $(_BuildConfig)
  nameSuffix: wasm
  isOfficialBuild: false
- extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml
+ extraStepsTemplate: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml
  extraStepsParameters:
- rootFolder: '$(Build.SourcesDirectory)/artifacts/'
- includeRootFolder: true
- displayName: Browser Wasm Artifacts
- artifactName: BrowserWasm
- archiveType: zip
- archiveExtension: .zip
+ configForBuild: Release
 
  #run mono wasm microbenchmarks perf job
  - template: /eng/pipelines/common/platform-matrix.yml
  parameters:
  jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs?
- buildConfig: release
+ buildConfig: Release
  runtimeFlavor: wasm
  platforms:
  - Linux_x64
@@ -97,13 +92,14 @@ jobs:
  - template: /eng/pipelines/common/platform-matrix.yml
  parameters:
  jobtemplate: /eng/pipelines/coreclr/templates/perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs?
- buildconfig: release
+ buildconfig: Release
  runtimeflavor: wasm
  platforms:
  - linux_x64
  jobparameters:
  testgroup: perf
  livelibrariesbuildconfig: Release
+ skipLiveLibrariesDownload: true
  runtimetype: wasm
  codegentype: 'aot'
  projectfile: microbenchmarks.proj
@@ -131,22 +127,17 @@ jobs:
  - template: /eng/pipelines/common/platform-matrix.yml
  parameters:
  jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: release
+ buildConfig: Release
  runtimeFlavor: mono
  platforms:
  - Browser_wasm
  jobParameters:
  buildArgs: -s mono+libs+host+packs -c $(_BuildConfig)
  nameSuffix: wasm
  isOfficialBuild: false
- extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml
+ extraStepsTemplate: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml
  extraStepsParameters:
- rootFolder: '$(Build.SourcesDirectory)/artifacts/'
- includeRootFolder: true
- displayName: Browser Wasm Artifacts
- artifactName: BrowserWasm
- archiveType: zip
- archiveExtension: .zip
+ configForBuild: Release
 
  # build mono for AOT
  - template: /eng/pipelines/common/platform-matrix.yml
@@ -326,6 +317,7 @@ jobs:
  jobParameters:
  testGroup: perf
  liveLibrariesBuildConfig: Release
+ skipLiveLibrariesDownload: true
  runtimeType: wasm
  codeGenType: 'wasm'
  projectFile: microbenchmarks.proj
@@ -345,6 +337,7 @@ jobs:
  jobparameters:
  testgroup: perf
  livelibrariesbuildconfig: Release
+ skipLiveLibrariesDownload: true
  runtimetype: wasm
  codegentype: 'aot'
  projectfile: microbenchmarks.proj
@@ -485,6 +478,7 @@ jobs:
  jobParameters:
  testGroup: perf
  liveLibrariesBuildConfig: Release
+ skipLiveLibrariesDownload: true
  runtimeType: wasm
  projectFile: blazor_perf.proj
  runKind: blazor_scenarios

eng/pipelines/coreclr/templates/perf-job.yml

@@ -19,6 +19,7 @@ parameters:
  pgoRunType: ''
  javascriptEngine: 'NoJS'
  iOSLlvmBuild: 'False'
+ skipLiveLibrariesDownload: false
 
 ### Perf job
 
@@ -50,9 +51,9 @@ jobs:
  iosLlvmBuild: ${{ parameters.iosLlvmBuild }}
  # Test job depends on the corresponding build job
  dependsOn:
- - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono')) }}:
+ - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'wasm')) }}:
  - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
- - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
+ - ${{ if and(ne(parameters.liveLibrariesBuildConfig, ''), eq(parameters.skipLiveLibrariesDownload, 'false')) }}:
  - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }}
  - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}:
  - ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
@@ -98,7 +99,7 @@ jobs:
  # Extra steps that will be passed to the performance template and run before sending the job to helix (all of which is done in the template)
 
  # Optionally download live-built libraries
- - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
+ - ${{ if and(ne(parameters.liveLibrariesBuildConfig, ''), eq(parameters.skipLiveLibrariesDownload, 'false')) }}:
  - template: /eng/pipelines/common/download-artifact-step.yml
  parameters:
  unpackFolder: $(librariesDownloadDir)
@@ -108,15 +109,15 @@ jobs:
  displayName: 'live-built libraries'
 
  # Download coreclr
- - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono')) }}:
+ - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'wasm')) }}:
  - template: /eng/pipelines/common/download-artifact-step.yml
  parameters:
  unpackFolder: $(buildProductRootFolderPath)
  artifactFileName: '$(buildProductArtifactName)$(archiveExtension)'
  artifactName: '$(buildProductArtifactName)'
  displayName: 'Coreclr product build'
 
- # Download mono 
+ # Download mono
  - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}:
  - template: /eng/pipelines/common/download-artifact-step.yml
  parameters:
@@ -135,7 +136,11 @@ jobs:
  artifactName: BrowserWasm
  displayName: BrowserWasm
 
- - script: "mkdir $(librariesDownloadDir)/bin/wasm;unzip -o $(librariesDownloadDir)/BrowserWasm/artifacts/packages/Release/Shipping/Microsoft.NETCore.App.Runtime.Mono.browser-wasm.7.0.0-ci.nupkg data/* runtimes/* -d $(librariesDownloadDir)/bin/wasm;cp src/mono/wasm/test-main.js $(librariesDownloadDir)/bin/wasm/test-main.js;find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \\;"
+ - script: >-
+ mkdir -p $(librariesDownloadDir)/bin/wasm/data &&
+ cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-workload $(librariesDownloadDir)/bin/wasm &&
+ cp src/mono/wasm/test-main.js $(librariesDownloadDir)/bin/wasm/data/test-main.js &&
+ find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \;
  displayName: "Create wasm directory (Linux)"
 
  # Download mono AOT
@@ -201,7 +206,7 @@ jobs:
  # Create Core_Root
  - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg)
  displayName: Create Core_Root
- condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono'))
+ condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono'), ne('${{ parameters.runtimeType }}', 'wasm'))
 
  # Copy the runtime directory into the testhost folder to include OOBs.
  - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\runtime\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\shared\\Microsoft.NETCore.App\\7.0.0 /E /I /Y;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y;copy $(Build.SourcesDirectory)\\artifacts\\bin\\coreclr\\$(osGroup).$(archType).$(buildConfigUpper)\\corerun.exe $(Build.SourcesDirectory)\\.dotnet-mono\\shared\\Microsoft.NETCore.App\\7.0.0\\corerun.exe"

eng/pipelines/coreclr/templates/run-scenarios-job.yml

@@ -107,7 +107,7 @@ jobs:
  displayName: Run ci setup script (Linux)
  condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
  # copy wasm packs if running on wasm
- - script: cp -r $(librariesDownloadDir)/BrowserWasm/artifacts/bin/microsoft.netcore.app.runtime.browser-wasm $(PayloadDirectory);cp -r $(librariesDownloadDir)/BrowserWasm/artifacts/bin/microsoft.netcore.app.ref $(PayloadDirectory)
+ - script: cp -r $(librariesDownloadDir)/BrowserWasm/staging/microsoft.netcore.app.runtime.browser-wasm $(PayloadDirectory) && cp -r $(librariesDownloadDir)/BrowserWasm/staging/microsoft.netcore.app.ref $(PayloadDirectory)
  displayName: Copy browserwasm and runtime ref packs
  condition: and(succeeded(), eq('${{ parameters.runtimeType }}', 'wasm'))
  # copy scenario support files

eng/testing/performance/performance-setup.sh

@@ -280,20 +280,9 @@ if [[ -n "$wasm_runtime_loc" ]]; then
  using_wasm=true
  wasm_dotnet_path=$payload_directory/dotnet-wasm
  mv $wasm_runtime_loc $wasm_dotnet_path
- # install emsdk, $source_directory/src/mono/wasm/ has the nuget.config with require feed. EMSDK may be available in the payload in a different directory, should visit this install to avoid deplicated payload.
- pushd $source_directory/src/mono/wasm/
- make provision-wasm
- EMSDK_PATH = $source_directory/src/mono/wasm/emsdk
- popd
- # wasm aot and interpreter need some source code from dotnet\runtime repo
- rsync -aq --progress $source_directory/* $wasm_dotnet_path --exclude Payload --exclude docs --exclude src/coreclr --exclude src/tests --exclude artifacts/obj --exclude artifacts/log --exclude artifacts/tests --exclude __download__
- # copy wasm build drop to the location that aot and interpreter build expects
- rsync -a --progress $wasm_dotnet_path/artifacts/BrowserWasm/artifacts/* $wasm_dotnet_path/artifacts
- rm -r $wasm_dotnet_path/artifacts/BrowserWasm/artifacts
+ extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmEngine /home/helixbot/.jsvu/$javascript_engine --cli \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/dotnet-workload/dotnet --wasmDataDir \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/data"
  if [[ "$wasmaot" == "true" ]]; then
- extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmEngine /home/helixbot/.jsvu/$javascript_engine --runtimeSrcDir \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm --aotcompilermode wasm --buildTimeout 3600"
- else
- extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmEngine /home/helixbot/.jsvu/$javascript_engine --runtimeSrcDir \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm"
+ extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --aotcompilermode wasm --buildTimeout 3600"
  fi
 fi