Build script changes for SxS willow instances

Sharing build script changes to support development in SxS willow environment.

`configure.ps1` script is now able to identify the latest willow instance and deduce VS install dir and msbuild path. Configuration parameters are store in `artifacts\configure.json` file so that build or test script don't need to run in the same PS session.

Fixed VC++ project system interop assembly dependency.

Optional. Set VSIX version in manifest to 3.6.99.99 to be able to publish locally built extension to experimental instance. Otherwise this fails due to version conflict. This change is safe as CI overwrites the version string in the manifest on S&P build step.

Code cleanup to fix:
- Use of unapproved verbs
- Use of redundant switch

Intentionally setting VSIX manifest version to 99.99.99

build.ps1

@@ -11,9 +11,6 @@ Release label to use for package and assemblies versioning (zlocal by default)
 .PARAMETER BuildNumber
 Build number to use for package and assemblies versioning (auto-generated if not provided)
 
-.PARAMETER SkipRestore
-Builds without restoring first
-
 .PARAMETER MSPFXPath
 Path to a code signing certificate for delay-sigining (optional)
 
@@ -29,11 +26,8 @@ Skips building binaries targeting Visual Studio "14" (released as Visual Studio
 .PARAMETER SkipVS15
 Skips building binaries targeting Visual Studio "15"
 
-.PARAMETER SkipILMerge
-Skips creating an ILMerged nuget.exe
-
 .PARAMETER Fast
-Runs minimal incremental build. ILMerge and end-to-end packaging steps skipped.
+Runs minimal incremental build. Skips end-to-end packaging step.
 
 .PARAMETER CI
 Indicates the build script is invoked from CI
@@ -64,8 +58,6 @@ param (
     [string]$ReleaseLabel = 'zlocal',
     [Alias('n')]
     [int]$BuildNumber,
-    [Alias('sr')]
-    [switch]$SkipRestore,
     [Alias('mspfx')]
     [string]$MSPFXPath,
     [Alias('nugetpfx')]
@@ -76,8 +68,6 @@ param (
     [switch]$SkipVS14,
     [Alias('s15')]
     [switch]$SkipVS15,
-    [Alias('si')]
-    [switch]$SkipILMerge,
     [Alias('f')]
     [switch]$Fast,
     [switch]$CI
@@ -123,54 +113,39 @@ Invoke-BuildStep 'Cleaning artifacts' {
     -skip:($Fast -or $SkipXProj) `
     -ev +BuildErrors
 
-# Restoring tools required for build
-Invoke-BuildStep 'Restoring solution packages' { Restore-SolutionPackages } `
-    -skip:$SkipRestore `
-    -ev +BuildErrors
-
 Invoke-BuildStep 'Enabling delay-signing' {
-        param($MSPFXPath, $NuGetPFXPath)
         Enable-DelaySigning $MSPFXPath $NuGetPFXPath
     } `
-    -args $MSPFXPath, $NuGetPFXPath `
     -skip:((-not $MSPFXPath) -and (-not $NuGetPFXPath)) `
     -ev +BuildErrors
 
 Invoke-BuildStep 'Building NuGet.Core projects' {
-        param($Configuration, $ReleaseLabel, $BuildNumber, $SkipRestore)
-        Build-CoreProjects $Configuration $ReleaseLabel $BuildNumber -SkipRestore:$SkipRestore
+        Build-CoreProjects $Configuration $ReleaseLabel $BuildNumber -CI:$CI
     } `
-    -args $Configuration, $ReleaseLabel, $BuildNumber, $SkipRestore `
     -skip:$SkipXProj `
     -ev +BuildErrors
 
 ## Building the VS15 Tooling solution
 Invoke-BuildStep 'Building NuGet.Clients projects - VS15 Toolset' {
-        param($Configuration, $ReleaseLabel, $BuildNumber, $SkipRestore)
-        Build-ClientsProjects $Configuration $ReleaseLabel $BuildNumber -ToolsetVersion 15 -SkipRestore:$SkipRestore
+        Build-ClientsProjects $Configuration $ReleaseLabel $BuildNumber -ToolsetVersion 15
     } `
-    -args $Configuration, $ReleaseLabel, $BuildNumber, $SkipRestore `
     -skip:$SkipVS15 `
     -ev +BuildErrors
 
 ## Building the VS14 Tooling solution
 Invoke-BuildStep 'Building NuGet.Clients projects - VS14 Toolset' {
-        param($Configuration, $ReleaseLabel, $BuildNumber, $SkipRestore)
-        Build-ClientsProjects $Configuration $ReleaseLabel $BuildNumber -ToolsetVersion 14 -SkipRestore:$SkipRestore
+        Build-ClientsProjects $Configuration $ReleaseLabel $BuildNumber -ToolsetVersion 14
     } `
-    -args $Configuration, $ReleaseLabel, $BuildNumber, $SkipRestore `
     -skip:$SkipVS14 `
     -ev +BuildErrors
 
-Invoke-BuildStep 'Creating NuGet.Clients packages - VS14 Toolset' {
-        param($Configuration, $MSPFXPath, $SkipILMerge)
-        Build-ClientsPackages $Configuration $ReleaseLabel $BuildNumber -ToolsetVersion 14 -KeyFile $MSPFXPath -SkipILMerge:$SkipILMerge
+Invoke-BuildStep 'Publishing NuGet.Clients packages - VS14 Toolset' {
+        Publish-ClientsPackages $Configuration $ReleaseLabel $BuildNumber -ToolsetVersion 14 -KeyFile $MSPFXPath
     } `
-    -args $Configuration, $MSPFXPath, $SkipILMerge `
     -skip:($Fast -or $SkipVS14) `
     -ev +BuildErrors
 
-Invoke-BuildStep 'Creating the VS14 EndToEnd test package' {
+Invoke-BuildStep 'Publishing the VS14 EndToEnd test package' {
         param($Configuration)
         $EndToEndScript = Join-Path $PSScriptRoot scripts\cibuild\CreateEndToEndTestPackage.ps1 -Resolve
         $OutDir = Join-Path $Artifacts VS14
@@ -180,7 +155,7 @@ Invoke-BuildStep 'Creating the VS14 EndToEnd test package' {
     -skip:($Fast -or $SkipVS14) `
     -ev +BuildErrors
 
-Invoke-BuildStep 'Creating the VS15 EndToEnd test package' {
+Invoke-BuildStep 'Publishing the VS15 EndToEnd test package' {
         param($Configuration)
         $EndToEndScript = Join-Path $PSScriptRoot scripts\cibuild\CreateEndToEndTestPackage.ps1 -Resolve
         $OutDir = Join-Path $Artifacts VS15

build/common.props

@@ -18,10 +18,6 @@
     <ArtifactRoot>$(EnlistmentRoot)\artifacts</ArtifactRoot>
     <ArtifactRoot>$([System.IO.Path]::GetFullPath( $(ArtifactRoot) ))</ArtifactRoot>
 
-    <VisualStudioRegistryKey>HKEY_CURRENT_USER\Software\Microsoft\VisualStudio\$(VisualStudioVersion)_Config\InstallDir</VisualStudioRegistryKey>
-    <VisualStudioDefaultInstallDir>$(ProgramFiles)\Microsoft Visual Studio $(VisualStudioVersion)\Common7\IDE\</VisualStudioDefaultInstallDir>
-    <VisualStudioInstallDir>$([MSBuild]::GetRegistryValue($(VisualStudioRegistryKey), $(VisualStudioDefaultInstallDir))</VisualStudioInstallDir>
-
     <OutDirFx>$(VisualStudioVersion)\$(Configuration)\</OutDirFx>
     <OutputPath>$(ArtifactRoot)\$(MSBuildProjectName)\$(OutDirFx)</OutputPath>
     <OutputPath>$([System.IO.Path]::GetFullPath( $(OutputPath) ))\</OutputPath>

build/common.ps1

@@ -8,16 +8,11 @@ $DefaultMSBuildVersion = 15
 $PackageReleaseVersion = "3.6.0"
 
 $NuGetClientRoot = Split-Path -Path $PSScriptRoot -Parent
-
-# allow this to work for scripts/funcTests
-if ((Split-Path -Path $PSScriptRoot -Leaf) -eq "scripts") {
-    $NuGetClientRoot = Split-Path -Path $NuGetClientRoot -Parent
-}
-
-$CLIRoot = Join-Path $NuGetClientRoot 'cli'
+$CLIRoot = Join-Path $NuGetClientRoot cli
 $Nupkgs = Join-Path $NuGetClientRoot nupkgs
 $Artifacts = Join-Path $NuGetClientRoot artifacts
 $ReleaseNupkgs = Join-Path $Artifacts ReleaseNupkgs
+$ConfigureJson = Join-Path $Artifacts configure.json
 
 $DotNetExe = Join-Path $CLIRoot 'dotnet.exe'
 $NuGetExe = Join-Path $NuGetClientRoot '.nuget\nuget.exe'
@@ -68,7 +63,7 @@ Function Error-Log {
         Write-Error "[$(Trace-Time)]`t$ErrorMessage"
     }
     else {
-        Write-Error "[$(Trace-Time)]`t$ErrorMessage" -ErrorAction Stop
+        Write-Error "[$(Trace-Time)]`t[FATAL] $ErrorMessage" -ErrorAction Stop
     }
 }
 
@@ -101,7 +96,8 @@ Function Invoke-BuildStep {
         [Alias('args')]
         [Object[]]$Arguments,
         [Alias('skip')]
-        [switch]$SkipExecution
+        [switch]$SkipExecution,
+        [switch]$Critical
     )
     if (-not $SkipExecution) {
         if ($env:TEAMCITY_VERSION) {
@@ -148,10 +144,9 @@ Function Update-Submodules {
     param(
         [switch]$Force
     )
-
-    $SubmodulesDir = Join-Path $NuGetClientRoot submodules -Resolve
-    $Submodules = gci $SubmodulesDir -ea Ignore
-    if ($Force -or -not $Submodules) {
+    $Submodules = Join-Path $NuGetClientRoot submodules -Resolve
+    $GitAttributes = gci $Submodules\* -Filter '.gitattributes' -r -ea Ignore
+    if ($Force -or -not $GitAttributes) {
         $opts = 'submodule', 'update'
         $opts += '--init'
         if (-not $VerbosePreference) {
@@ -170,24 +165,25 @@ Function Install-NuGet {
     param(
         [switch]$Force
     )
-
     if ($Force -or -not (Test-Path $NuGetExe)) {
         Trace-Log 'Downloading nuget.exe'
         wget https://dist.nuget.org/win-x86-commandline/latest-prerelease/nuget.exe -OutFile $NuGetExe
     }
+
+    # Display nuget info
+    & $NuGetExe locals all -list -verbosity detailed
 }
 
 Function Install-DotnetCLI {
     [CmdletBinding()]
     param(
         [switch]$Force
     )
-
     $env:DOTNET_HOME=$CLIRoot
     $env:DOTNET_INSTALL_DIR=$NuGetClientRoot
 
     if ($Force -or -not (Test-Path $DotNetExe)) {
-        Trace-Log 'Downloading Dotnet CLI'
+        Trace-Log 'Downloading .NET CLI'
 
         New-Item -ItemType Directory -Force -Path $CLIRoot | Out-Null
 
@@ -257,21 +253,33 @@ Function Test-MSBuildVersionPresent {
     Test-Path $MSBuildExe
 }
 
-$MSBuildExe = Get-MSBuildExe
-Set-Alias msbuild $MSBuildExe
-
-$VS14Installed = Test-MSBuildVersionPresent -MSBuildVersion 14
-$VS15Installed = Test-MSBuildVersionPresent -MSBuildVersion 15
-
-function Test-BuildEnvironment {
+Function Test-BuildEnvironment {
+    [CmdletBinding()]
     param(
         [switch]$CI
     )
+    if (-not (Test-Path $ConfigureJson)) {
+        Error-Log 'Build environment is not configured. Please run configure.ps1 first.' -Fatal
+    }
+
     $Installed = (Test-Path $DotNetExe) -and (Test-Path $NuGetExe)
     if (-not $Installed) {
         Error-Log 'Build environment is not configured. Please run configure.ps1 first.' -Fatal
     }
 
+    $script:ConfigureObject = Get-Content $ConfigureJson -Raw | ConvertFrom-Json
+    Set-Variable MSBuildExe -Value $ConfigureObject.BuildTools.MSBuildExe -Scope Script -Force
+    Set-Alias msbuild $script:MSBuildExe -Scope Script -Force
+    Set-Variable BuildToolsets -Value $ConfigureObject.Toolsets -Scope Script -Force
+
+    $script:VS14Installed = ($BuildToolsets | where vs14 -ne $null)
+    $script:VS15Installed = ($BuildToolsets | where vs15 -ne $null)
+
+    $ConfigureObject |
+         select -expand envvars -ea Ignore |
+         %{ $_.psobject.properties } |
+         %{ Set-Item -Path "env:$($_.Name)" -Value $_.Value }
+
     if ($CI) {
         # Explicitly add cli to environment PATH
         # because dotnet-install script runs in configure.ps1 in previous build step
@@ -376,7 +384,7 @@ Function Clear-Artifacts {
     param()
     if( Test-Path $Artifacts) {
         Trace-Log 'Cleaning the Artifacts folder'
-        Remove-Item $Artifacts\* -Recurse -Force
+        Remove-Item $Artifacts\* -Recurse -Force -Exclude 'configure.json'
     }
 }
 
@@ -549,7 +557,8 @@ Function Build-CoreProjects {
         [string]$Configuration = $DefaultConfiguration,
         [string]$ReleaseLabel = $DefaultReleaseLabel,
         [int]$BuildNumber = (Get-BuildNumber),
-        [switch]$SkipRestore
+        [switch]$SkipRestore,
+        [switch]$CI
     )
     $XProjectsLocation = Join-Path $NuGetClientRoot src\NuGet.Core -Resolve
     $xprojects = Find-XProjects $XProjectsLocation
@@ -561,9 +570,11 @@ Function Build-CoreProjects {
     # Build .nupkgs for MyGet (which have release label and build number)
     $xprojects | Invoke-DotnetPack -config $Configuration -label $ReleaseLabel -build $BuildNumber -out $Nupkgs
 
-    # Build .nupkgs for release (which have no build number). This will re-use the build from the last
-    # step because the --build-base-path is the same.
-    $xprojects | Invoke-DotnetPack -config $Configuration -label $ReleaseLabel -out $ReleaseNupkgs
+    if ($CI) {
+        # Build .nupkgs for release (which have no build number). This will re-use the build from the last
+        # step because the --build-base-path is the same.
+        $xprojects | Invoke-DotnetPack -config $Configuration -label $ReleaseLabel -out $ReleaseNupkgs
+    }
 
     # Publish NuGet.CommandLine.XPlat
     $PublishLocation = Join-Path $Artifacts "NuGet.CommandLine.XPlat\publish"
@@ -588,7 +599,7 @@ Function Pack-NuGetBuildTasksPack {
 
     Remove-Item $Nupkgs\NuGet.Build.Tasks.Pack*
     Remove-Item $ReleaseNupkgs\NuGet.Build.Tasks.Pack*
-    
+
     $prereleaseNupkgVersion = "$PackageReleaseVersion-$ReleaseLabel-$BuildNumber"
     if ($ReleaseLabel -Ne 'rtm') {
         $releaseNupkgVersion = "$PackageReleaseVersion-$ReleaseLabel"
@@ -598,7 +609,7 @@ Function Pack-NuGetBuildTasksPack {
 
     $PackProjectLocation = Join-Path $NuGetClientRoot src\NuGet.Core\NuGet.Build.Tasks.Pack
     $PackBuildTaskNuspecLocation = Join-Path $PackProjectLocation NuGet.Build.Tasks.Pack.nuspec
-    
+
     Build-NuGetPackage `
             -NuspecPath $PackBuildTaskNuspecLocation `
             -BasePath $PackProjectLocation `
@@ -795,16 +806,15 @@ Function Build-ClientsProjects {
         -IsSolution
 }
 
-Function Build-ClientsPackages {
+Function Publish-ClientsPackages {
     [CmdletBinding()]
     param(
         [string]$Configuration = $DefaultConfiguration,
         [string]$ReleaseLabel = $DefaultReleaseLabel,
         [int]$BuildNumber = (Get-BuildNumber),
         [ValidateSet(14,15)]
         [int]$ToolsetVersion = $DefaultMSBuildVersion,
-        [string]$KeyFile,
-        [switch]$SkipILMerge
+        [string]$KeyFile
     )
 
     $prereleaseNupkgVersion = "$PackageReleaseVersion-$ReleaseLabel-$BuildNumber"
@@ -814,7 +824,6 @@ Function Build-ClientsPackages {
         $releaseNupkgVersion = "$PackageReleaseVersion"
     }
 
-    if (-not $SkipILMerge){
         $exeProjectDir = [io.path]::combine($NuGetClientRoot, "src", "NuGet.Clients", "NuGet.CommandLine")
         $exeProject = Join-Path $exeProjectDir "NuGet.CommandLine.csproj"
         $exeNuspec = Join-Path $exeProjectDir "NuGet.CommandLine.nuspec"
@@ -835,7 +844,7 @@ Function Build-ClientsPackages {
             -OutputDir $exeOutputDir `
             -KeyFile $KeyFile
 
-        Build-NuGetPackage `
+        New-NuGetPackage `
             -NuspecPath $exeNuspec `
             -BasePath $exeOutputDir `
             -OutputDir $Nupkgs `
@@ -857,7 +866,7 @@ Function Build-ClientsPackages {
             -OutputDir $exeOutputDir `
             -KeyFile $KeyFile
 
-        Build-NuGetPackage `
+        New-NuGetPackage `
             -NuspecPath $exeNuspec `
             -BasePath $exeOutputDir `
             -OutputDir $ReleaseNupkgs `
@@ -873,37 +882,47 @@ Function Build-ClientsPackages {
 
     Copy-Item -Path "${projectInstallPs1}" -Destination "${projectInputDir}"
 
-    Build-NuGetPackage `
+    New-NuGetPackage `
         -NuspecPath $projectNuspec `
         -BasePath $projectInputDir `
         -OutputDir $Nupkgs `
         -Version $prereleaseNupkgVersion `
         -Configuration $Configuration
 
     # Pack the NuGet.VisualStudio project with just the release label.
-    Build-NuGetPackage `
+    New-NuGetPackage `
         -NuspecPath $projectNuspec `
         -BasePath $projectInputDir `
         -OutputDir $ReleaseNupkgs `
         -Version $releaseNupkgVersion `
         -Configuration $Configuration
 }
 
-Function Build-NuGetPackage {
+Function New-NuGetPackage {
+    [CmdletBinding()]
     param(
         [string]$NuspecPath,
         [string]$BasePath,
         [string]$OutputDir,
         [string]$Version,
         [string]$Configuration=$DefaultConfiguration
-	)
+    )
 
     $opts = 'pack', $NuspecPath
     $opts += '-BasePath', $BasePath
     $opts += '-OutputDirectory', $OutputDir
     $opts += '-Symbols'
     $opts += '-Version', $Version
     $opts += '-Properties', "Configuration=$Configuration"
+
+    if ($VerbosePreference) {
+        $opts += '-verbosity', 'detailed'
+    }
+    else {
+        $opts += '-verbosity', 'quiet'
+    }
+
+    Trace-Log "$NuGetExe $opts"
     & $NuGetExe $opts
 }