From c36a54ed3308d1516ffe1a86b9086c42e4ca996f Mon Sep 17 00:00:00 2001 From: AR-May <67507805+AR-May@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:03:38 +0200 Subject: [PATCH] Add a job for experimental Framework MSBuild insertion to a pipeline (#9309) Context We need to add an extra job to .exp-insertions.yml pipeline for creating experimental Framework MSBuild. Changes Made Pipeline refactored, a job added Fixed bug in deploy script code path for Framework MSBuild Testing Manual run of the pipeline --- .exp-insertions.yml | 253 +++++++++++++++++++++++++------------ scripts/Deploy-MSBuild.ps1 | 24 ++-- 2 files changed, 185 insertions(+), 92 deletions(-) diff --git a/.exp-insertions.yml b/.exp-insertions.yml index e3e22653a53..b3d5d4e5f49 100644 --- a/.exp-insertions.yml +++ b/.exp-insertions.yml @@ -1,13 +1,19 @@ -# Pipeline creates a dotnet with experimental msbuild dlls. +# Pipeline creates experimental msbuild insertions. trigger: none # Prevents this pipeline from triggering on check-ins pr: none # don't run this on PR as well parameters: - # Dotnet installer channel from where to take the latest dotnet bits. + # Dotnet installer channel from which to take the latest dotnet bits. - name: DotnetInstallerChannel displayName: Dotnet installer channel + type: string + default: 'none' + # VS version for which to take the latest Retail MSBuild bits. + - name: VSVersionName + displayName: VS Version type: string + default: 'none' # Branch from the MSBuild Build CI pipeline. Default: main # Top run for the branch would be used to create an experimental insertion. - name: MSBuildBranch @@ -23,105 +29,192 @@ parameters: variables: - name: _MsBuildCiPipelineId value: 9434 + - name: _MSBuildConfigFilePath + value: "config/batmon/Q-Prod-Co3/Coordinator/ToolsReleaseConfig-GeneralPublic.json" + - name: VSVersion + value: ${{parameters.VSVersionName}} pool: vmImage: windows-latest -steps: -- powershell: | - mkdir '$(System.ArtifactsDirectory)/installer' - - $dotnetChannel = '${{parameters.DotnetInstallerChannel}}' - $sdks = "dotnet-sdk-win-x64.zip", "dotnet-sdk-linux-x64.tar.gz" - - foreach ($sdk in $sdks) - { - Write-Host "Downloading dotnet $sdk from channel $dotnetChannel" - Invoke-WebRequest ` - -Uri "https://aka.ms/dotnet/$dotnetChannel/daily/$sdk" ` - -OutFile "$(System.ArtifactsDirectory)/installer/$sdk" - } - mkdir '$(Pipeline.Workspace)/artifacts' - - displayName: Download latest dotnet sdks +resources: + repositories: + - repository: CloudBuildConfig + type: git + name: CloudBuild/CloudBuildConfig + endpoint: CloudBuild_Test + ref: refs/heads/main -# Download latest build artifacts for a branch from MSBuild Build CI -- ${{ if eq(parameters.MSBuildBuildID, 'default') }}: - - task: DownloadBuildArtifacts@1 - inputs: - buildType: specific - project: DevDiv - pipeline: $(_MsBuildCiPipelineId) - buildVersionToDownload: latestFromBranch - branchName: '${{parameters.MSBuildBranch}}' - artifactName: bin - downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin' - itemPattern: "MSBuild.Bootstrap/**" - displayName: Download latest msbuild from branch +jobs: +- job: CreateExpDotnet + displayName: Create Experimental Dotnet + condition: ne('${{ parameters.DotnetInstallerChannel }}', 'none') + steps: + - powershell: | + mkdir '$(System.ArtifactsDirectory)/installer' + + $dotnetChannel = '${{parameters.DotnetInstallerChannel}}' + $sdks = "dotnet-sdk-win-x64.zip", "dotnet-sdk-linux-x64.tar.gz" + + foreach ($sdk in $sdks) + { + Write-Host "Downloading dotnet $sdk from channel $dotnetChannel" + Invoke-WebRequest ` + -Uri "https://aka.ms/dotnet/$dotnetChannel/daily/$sdk" ` + -OutFile "$(System.ArtifactsDirectory)/installer/$sdk" + } + mkdir '$(Pipeline.Workspace)/artifacts' + displayName: Download latest dotnet sdks -# Download build artifacts for MSBuild Build CI specific build -- ${{ if ne(parameters.MSBuildBuildID, 'default') }}: - task: DownloadBuildArtifacts@1 inputs: buildType: specific project: DevDiv pipeline: $(_MsBuildCiPipelineId) - buildVersionToDownload: specific - buildId: ${{parameters.MSBuildBuildID}} + ${{ if eq(parameters.MSBuildBuildID, 'default') }}: + buildVersionToDownload: latestFromBranch + branchName: '${{parameters.MSBuildBranch}}' + ${{ else }}: + buildVersionToDownload: specific + buildId: ${{parameters.MSBuildBuildID}} artifactName: bin downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin' - itemPattern: "MSBuild.Bootstrap/**" - displayName: Download specified msbuild build - -- powershell: | - $sdk = "dotnet-sdk-win-x64" + displayName: Download msbuild artifacts - Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.zip" - Expand-Archive "$(System.ArtifactsDirectory)/installer/$sdk.zip" -DestinationPath "$(Pipeline.Workspace)/exp-dotnet/$sdk" + - powershell: | + $sdk = "dotnet-sdk-win-x64" + + Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.zip" + Expand-Archive "$(System.ArtifactsDirectory)/installer/$sdk.zip" -DestinationPath "$(Pipeline.Workspace)/exp-dotnet/$sdk" - $dotnetDirectory = Get-ChildItem -Directory -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk" - $dotnetVersion = $dotnetDirectory.Name - Write-Host "Detected dotnet version: $dotnetVersion" + $dotnetDirectory = Get-ChildItem -Directory -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk" + $dotnetVersion = $dotnetDirectory.Name + Write-Host "Detected dotnet version: $dotnetVersion" - Write-Host "Updating MSBuild dlls." - $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 ` - -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" ` - -bootstrapDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin/MSBuild.Bootstrap" ` - -configuration Release ` - -makeBackup $false + Write-Host "Updating MSBuild dlls." + $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 ` + -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" ` + -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" ` + -configuration Release ` + -makeBackup $false - Write-Host "Compressing dotnet sdk files" - Get-ChildItem -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/$sdk.zip" + Write-Host "Compressing dotnet sdk files" + Get-ChildItem -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/$sdk.zip" - displayName: Dogfood msbuild dlls to dotnet sdk win-x64 + displayName: Dogfood msbuild dlls to dotnet sdk win-x64 -- powershell: | - $sdk = "dotnet-sdk-linux-x64" + - powershell: | + $sdk = "dotnet-sdk-linux-x64" - mkdir "$(Pipeline.Workspace)/exp-dotnet/$sdk" + mkdir "$(Pipeline.Workspace)/exp-dotnet/$sdk" - Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.tar.gz" - tar -xzvf "$(System.ArtifactsDirectory)/installer/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" + Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.tar.gz" + tar -xzvf "$(System.ArtifactsDirectory)/installer/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" - $dotnetDirectory = Get-ChildItem -Directory -Path $(Pipeline.Workspace)/exp-dotnet/$sdk/sdk - $dotnetVersion = $dotnetDirectory.Name - Write-Host "Detected dotnet version: $dotnetVersion" + $dotnetDirectory = Get-ChildItem -Directory -Path $(Pipeline.Workspace)/exp-dotnet/$sdk/sdk + $dotnetVersion = $dotnetDirectory.Name + Write-Host "Detected dotnet version: $dotnetVersion" - Write-Host "Updating MSBuild dlls." - $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 ` - -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" ` - -bootstrapDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin/MSBuild.Bootstrap" ` - -configuration Release ` - -makeBackup $false + Write-Host "Updating MSBuild dlls." + $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 ` + -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" ` + -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" ` + -configuration Release ` + -makeBackup $false - Write-Host "Compressing dotnet sdk files" - tar -czvf "$(Pipeline.Workspace)/artifacts/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" . - displayName: Dogfood msbuild dlls to dotnet sdk linux-x64 - -- task: PublishPipelineArtifact@1 - inputs: - targetPath: '$(Pipeline.Workspace)/artifacts' - artifactName: ExperimentalDotnet - parallel: true - condition: always() - displayName: Publish crank assests artifacts + Write-Host "Compressing dotnet sdk files" + tar -czvf "$(Pipeline.Workspace)/artifacts/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" . + displayName: Dogfood msbuild dlls to dotnet sdk linux-x64 + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Pipeline.Workspace)/artifacts' + artifactName: ExperimentalDotnet + parallel: true + condition: always() + displayName: Publish crank assests artifacts + + +- job: CreateExpMSBuild + displayName: "Create Experimental MSBuild" + condition: ne('${{ parameters.VSVersionName }}', 'none') + steps: + - checkout: self + + - checkout: CloudBuildConfig + + - powershell: | + $json = (Get-Content "$(Build.SourcesDirectory)/CloudBuildConfig/$(_MSBuildConfigFilePath)" -Raw) | ConvertFrom-Json + $MSBuildDropPath = $json.Tools.MSBuild.Locations + Write-Host "##vso[task.setvariable variable=MSBuildDropPath]$MSBuildDropPath" + Write-Host "MSBuild Drop Path directory: $MSBuildDropPath" + displayName: Get Retail MSBuild Drop Path + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + - task: NuGetCommand@2 + displayName: Restore internal tools + inputs: + command: restore + feedsToUse: config + restoreSolution: '$(Build.SourcesDirectory)\DotNet-msbuild-Trusted\eng\common\internal\Tools.csproj' + nugetConfigPath: '$(Build.SourcesDirectory)\DotNet-msbuild-Trusted\NuGet.config' + restoreDirectory: '$(Build.SourcesDirectory)\DotNet-msbuild-Trusted\.packages' + + - powershell: | + mkdir "$(Pipeline.Workspace)/artifacts" + + $dropAppDirectory = Get-ChildItem -Directory -Path "$(Build.SourcesDirectory)/DotNet-msbuild-Trusted/.packages/drop.app" + $dropAppVersion = $dropAppDirectory.Name + Write-Host "Detected drop.exe version: $dropAppVersion" + + $dropExePath = "$(Build.SourcesDirectory)/DotNet-msbuild-Trusted/.packages/drop.app/$dropAppVersion/lib/net45/drop.exe" + Write-Host "Detected drop.exe path: $dropExePath" + + Write-Host "Downloading VS msbuild" + & "$dropExePath" get --patAuthEnvVar 'cloudbuild-token' -u "$(MSBuildDropPath)\$(VSVersion)" -d "$(System.ArtifactsDirectory)/VSMSBuildDrop" + Write-Host "Download of VS msbuild finished" + + Write-Host "Copying VS msbuild to $(Pipeline.Workspace)/VSMSBuild" + Copy-Item -Path "$(System.ArtifactsDirectory)/VSMSBuildDrop/*" -Destination "$(Pipeline.Workspace)/VSMSBuild" -Recurse + Write-Host "Copy of VS msbuild finished" + displayName: Download msbuild vs drop + env: + cloudbuild-token: $(cloudbuild-token) + + - task: DownloadBuildArtifacts@1 + inputs: + buildType: specific + project: DevDiv + pipeline: $(_MsBuildCiPipelineId) + ${{ if eq(parameters.MSBuildBuildID, 'default') }}: + buildVersionToDownload: latestFromBranch + branchName: '${{parameters.MSBuildBranch}}' + ${{ else }}: + buildVersionToDownload: specific + buildId: ${{parameters.MSBuildBuildID}} + artifactName: bin + downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin' + displayName: Download msbuild artifacts + + - powershell: | + Write-Host "Updating MSBuild dlls." + $(Build.SourcesDirectory)/DotNet-msbuild-Trusted/scripts/Deploy-MSBuild.ps1 ` + -destination "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)/MSBuild/Current/Bin" ` + -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" ` + -configuration Release ` + -makeBackup $false + + ls "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)" + Write-Host "Compressing msbuild files" + Get-ChildItem -Path "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/MSBuild.zip" + displayName: Dogfood msbuild dlls + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Pipeline.Workspace)/artifacts' + artifactName: ExperimentalMSBuild + parallel: true + condition: always() + displayName: Publish crank assests artifacts diff --git a/scripts/Deploy-MSBuild.ps1 b/scripts/Deploy-MSBuild.ps1 index 58966a34fdf..95e2726de9f 100644 --- a/scripts/Deploy-MSBuild.ps1 +++ b/scripts/Deploy-MSBuild.ps1 @@ -6,7 +6,7 @@ Param( [string] $configuration = "Debug", [ValidateSet('Core','Desktop', 'Detect', 'Full')] [string] $runtime = "Detect", - [string] $bootstrapDirectory = "", + [string] $binDirectory = "", [bool] $makeBackup = $true ) @@ -80,11 +80,11 @@ if ($runtime -eq "Desktop") { } # If bootstrap directory is not defined in parameters, use the default location -if ($bootstrapDirectory -eq "") { - $bootstrapDirectory = "artifacts\bin\MSBuild.Bootstrap" +if ($binDirectory -eq "") { + $binDirectory = "artifacts\bin" } -$bootstrapBinDirectory = "$bootstrapDirectory\$configuration\$targetFramework" +$bootstrapBinDirectory = "$binDirectory\MSBuild.Bootstrap\$configuration\$targetFramework" $filesToCopyToBin = @( FileToCopy "$bootstrapBinDirectory\Microsoft.Build.dll" @@ -116,8 +116,8 @@ $filesToCopyToBin = @( if ($runtime -eq "Desktop") { $runtimeSpecificFiles = @( - FileToCopy "artifacts\bin\Microsoft.Build.Conversion\$configuration\$targetFramework\Microsoft.Build.Conversion.Core.dll" - FileToCopy "artifacts\bin\Microsoft.Build.Engine\$configuration\$targetFramework\Microsoft.Build.Engine.dll" + FileToCopy "$binDirectory\Microsoft.Build.Conversion\$configuration\$targetFramework\Microsoft.Build.Conversion.Core.dll" + FileToCopy "$binDirectory\Microsoft.Build.Engine\$configuration\$targetFramework\Microsoft.Build.Engine.dll" FileToCopy "$bootstrapBinDirectory\Microsoft.Bcl.AsyncInterfaces.dll" FileToCopy "$bootstrapBinDirectory\Microsoft.Data.Entity.targets" @@ -152,14 +152,14 @@ if ($runtime -eq "Desktop") { $x86files = @( FileToCopy "$bootstrapBinDirectory\MSBuild.exe" FileToCopy "$bootstrapBinDirectory\MSBuild.exe.config" - FileToCopy "artifacts\bin\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.exe" - FileToCopy "artifacts\bin\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.pdb" + FileToCopy "$binDirectory\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.exe" + FileToCopy "$binDirectory\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.pdb" ) $amd64files = @( - FileToCopy "artifacts\bin\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe" - FileToCopy "artifacts\bin\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe.config" - FileToCopy "artifacts\bin\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.exe" - FileToCopy "artifacts\bin\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.pdb" + FileToCopy "$binDirectory\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe" + FileToCopy "$binDirectory\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe.config" + FileToCopy "$binDirectory\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.exe" + FileToCopy "$binDirectory\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.pdb" ) }