Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve installation of .NET for building NuGet.Client repository #5134

Merged
merged 3 commits into from
Apr 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<PackageVersion Include="MessagePack" Version="2.2.60" />
<PackageVersion Include="Microsoft.Build" Version="$(MicrosoftBuildVersion)" />
<PackageVersion Include="Microsoft.Build.Framework" Version="$(MicrosoftBuildVersion)" />
<PackageVersion Include="Microsoft.Build.Locator" Version="1.4.1" />
<PackageVersion Include="Microsoft.Build.Locator" Version="1.5.5" />
<PackageVersion Include="Microsoft.Build.Runtime" Version="$(MicrosoftBuildVersion)" />
<PackageVersion Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildVersion)" />
<PackageVersion Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildVersion)" />
Expand Down
80 changes: 7 additions & 73 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,84 +8,19 @@ while true ; do
esac
done

RESULTCODE=0

# Download the CLI install script to cli
echo "Installing dotnet CLI"
mkdir -p cli
curl -o cli/dotnet-install.sh -L https://dot.net/v1/dotnet-install.sh

if (( $? )); then
echo "Could not download 'dotnet-install.sh' script. Please check your network and try again!"
exit 1
fi

# Run install.sh for cli
chmod +x cli/dotnet-install.sh

# Get recommended version for bootstrapping testing version
cli/dotnet-install.sh -i cli -c 2.2 -nopath

if (( $? )); then
echo "The .NET CLI Install failed!!"
exit 1
# Run configure which installs the .NET SDK
. ./configure.sh
if [ $? -ne 0 ]; then
echo "configure.sh failed !!"
exit 1
fi

# Disable .NET CLI Install Lookup
DOTNET_MULTILEVEL_LOOKUP=0

DOTNET="$(pwd)/cli/dotnet"

# Let the dotnet cli expand and decompress first if it's a first-run
$DOTNET --info

# Get CLI Branches for testing
echo "dotnet msbuild build/config.props /restore:false /ConsoleLoggerParameters:Verbosity=Minimal;NoSummary;ForceNoAlign /nologo /target:GetCliBranchForTesting"

IFS=$'\n'
CMD_OUT_LINES=(`dotnet msbuild build/config.props /restore:false /ConsoleLoggerParameters:Verbosity=Minimal;NoSummary;ForceNoAlign /nologo /target:GetCliBranchForTesting`)
# Take only last the line which has the version information and strip all the spaces
DOTNET_BRANCHES=${CMD_OUT_LINES[-1]//[[:space:]]}
unset IFS

IFS=$';'
for DOTNET_BRANCH in ${DOTNET_BRANCHES[@]}
do
echo $DOTNET_BRANCH

IFS=$':'
ChannelAndVersion=($DOTNET_BRANCH)
Channel=${ChannelAndVersion[0]}
if [ ${#ChannelAndVersion[@]} -eq 1 ]
then
Version="latest"
else
Version=${ChannelAndVersion[1]}
fi
unset IFS

echo "Channel is: $Channel"
echo "Version is: $Version"
cli/dotnet-install.sh -i cli -c $Channel -v $Version -nopath

if (( $? )); then
echo "The .NET CLI Install for $DOTNET_BRANCH failed!!"
exit 1
fi
done

# Display .NET CLI info
$DOTNET --info

echo "================="

# init the repo
git submodule init
git submodule update

# clear caches
if [ "$CLEAR_CACHE" == "1" ]
then
if [ "$CLEAR_CACHE" == "1" ]; then
# echo "Clearing the nuget web cache folder"
# rm -r -f ~/.local/share/NuGet/*

Expand All @@ -96,6 +31,7 @@ fi
# restore packages
echo "dotnet msbuild build/bootstrap.proj /t:Restore"
dotnet msbuild build/bootstrap.proj /t:Restore

echo "dotnet msbuild build/build.proj /t:Restore /p:Configuration=Release /p:BuildNumber=1 /p:ReleaseLabel=beta"
dotnet msbuild build/build.proj /t:Restore /p:Configuration=Release /p:BuildNumber=1 /p:ReleaseLabel=beta

Expand All @@ -112,5 +48,3 @@ if [ $? -ne 0 ]; then
echo "Tests failed!!"
exit 1
fi

exit $RESULTCODE
5 changes: 5 additions & 0 deletions build/DotNetSdkVersions.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Each line represents arguments for the .NET SDK installer script (https://learn.microsoft.com/dotnet/core/tools/dotnet-install-script)
-Channel 7.0
-Channel 6.0 -Runtime dotnet
Copy link
Member

@zivkan zivkan Apr 19, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the purpose of the -runtime dotnet on the line, but not others?

We've had contributors using some IBM mainframe contribute to NuGet, but it uses the mono runtime (not Mono Project, that implements .NET Framework, but the mono runtime of the .NET Core App runtime). By specifying -runtime dotnet, is it going to cause problems for customers on platforms that the dotnet runtime doesn't have binaries for?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I found that some of our tests need the .NET 6 runtime but don't need the full SDK. I've experimented a little and hope to trim this down as soon as all of our tests target .NET 7 or .NET 8.

-Channel 5.0
Copy link
Member

@nkolev92 nkolev92 Jun 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need the 5.0 and 3.1 SDKs?
Aren't the runtimes sufficient for this?

We'll always build with the latest installed SDK anyways.

-Channel 3.1
80 changes: 31 additions & 49 deletions build/common.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -154,91 +154,73 @@ Function Update-Submodules {
Function Install-DotnetCLI {
[CmdletBinding()]
param(
[switch]$Force
[switch]$Force,
[switch]$SkipDotnetInfo
)
$MSBuildExe = Get-MSBuildExe

$CmdOutLines = ((& $msbuildExe $NuGetClientRoot\build\config.props /restore:false "/ConsoleLoggerParameters:Verbosity=Minimal;NoSummary;ForceNoAlign" /nologo /target:GetCliBranchForTesting) | Out-String).Trim()
$CliBranchListForTesting = ($CmdOutLines -split [Environment]::NewLine)[-1]
$CliBranchList = $CliBranchListForTesting -split ';'

$DotNetInstall = Join-Path $CLIRoot 'dotnet-install.ps1'

#If "-force" is specified, or dotnet.exe under cli folder doesn't exist, create cli folder and download dotnet-install.ps1 into cli folder.
if ($Force -or -not (Test-Path $DotNetExe)) {
Trace-Log "Downloading .NET CLI '$CliBranchList'"
Trace-Log "Downloading .NET CLI install script"

New-Item -ItemType Directory -Force -Path $CLIRoot | Out-Null

Invoke-WebRequest 'https://dot.net/v1/dotnet-install.ps1' -OutFile $DotNetInstall
}

if (-not ([string]::IsNullOrEmpty($env:DOTNET_SDK_VERSIONS))) {
Trace-Log "Using environment variable DOTNET_SDK_VERSIONS instead of DotNetSdkVersions.txt. Value: '$env:DOTNET_SDK_VERSIONS'"
$CliBranchList = $env:DOTNET_SDK_VERSIONS -Split ";"
} else {
$CliBranchList = (Get-Content -Path "$NuGetClientRoot\build\DotNetSdkVersions.txt")
}

ForEach ($CliBranch in $CliBranchList) {
$CliBranch = $CliBranch.trim()
$CliChannelAndVersion = $CliBranch -split ":"

# If version is not specified, use 'latest' as the version.
$Channel = $CliChannelAndVersion[0].trim()
if ($CliChannelAndVersion.count -eq 1) {
$Version = 'latest'
}
else {
$Version = $CliChannelAndVersion[1].trim()
}

$cli = @{
Root = $CLIRoot
Version = $Version
Channel = $Channel
if ($CliBranch.StartsWith("#") -or $CliBranch.Equals("")) {
continue
}

$DotNetExe = Join-Path $cli.Root 'dotnet.exe';

if ([Environment]::Is64BitOperatingSystem) {
$arch = "x64";
}
else {
$arch = "x86";
}

Trace-Log "$DotNetInstall -Channel $($cli.Channel) -InstallDir $($cli.Root) -Version $($cli.Version) -Architecture $arch -NoPath"
Trace-Log "$DotNetInstall $CliBranch -InstallDir $CLIRoot -Architecture $arch -NoPath"

& powershell $DotNetInstall -Channel $cli.Channel -InstallDir $cli.Root -Version $cli.Version -Architecture $arch -NoPath
& powershell $DotNetInstall $CliBranch -InstallDir $CLIRoot -Architecture $arch -NoPath
if ($LASTEXITCODE -ne 0)
{
throw "dotnet-install.ps1 exited with non-zero exit code"
}
}

if (-not (Test-Path $DotNetExe)) {
Error-Log "Unable to find dotnet.exe. The CLI install may have failed." -Fatal
}

if (-not (Test-Path $DotNetExe)) {
Error-Log "Unable to find dotnet.exe. The CLI install may have failed." -Fatal
}

if ($SkipDotnetInfo -ne $true) {
# Display build info
& $DotNetExe --info
if ($LASTEXITCODE -ne 0)
{
throw "dotnet --info exited with non-zero exit code"
}
}

# Install the 3.x runtime because our tests target netcoreapp2x
Trace-Log "$DotNetInstall -Runtime dotnet -Channel 3.1 -InstallDir $CLIRoot -NoPath"
# dotnet-install might make http requests that fail, but it handles those errors internally
# However, Invoke-BuildStep checks if any error happened, ever. Hence we need to run dotnet-install
# in a different process, to avoid treating their handled errors as build errors.
& powershell $DotNetInstall -Runtime dotnet -Channel 3.1 -InstallDir $CLIRoot -NoPath
& powershell $DotNetInstall -Runtime dotnet -Channel 5.0 -InstallDir $CLIRoot -NoPath

if ($LASTEXITCODE -ne 0)
{
throw "dotnet-install.ps1 exited with non-zero exit code"
}

# Display build info
& $DotNetExe --info
if ($LASTEXITCODE -ne 0)
{
throw "dotnet --info exited with non-zero exit code"

if ($env:CI -eq "true") {
Write-Host "##vso[task.setvariable variable=DOTNET_ROOT;isOutput=false;issecret=false;]$CLIRoot"
Write-Host "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP;isOutput=false;issecret=false;]0"
Write-Host "##vso[task.prependpath]$CLIRoot"
} else {
$env:DOTNET_ROOT=$CLIRoot
$env:DOTNET_MULTILEVEL_LOOKUP=0
if (-not $env:path.Contains($CLIRoot)) {
$env:path = $CLIRoot + ";" + $env:path
}
}
}

Expand Down
13 changes: 0 additions & 13 deletions build/config.props
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,6 @@
<!-- This branches are used for creating insertion PRs -->
<VsTargetBranch Condition="'$(IsEscrowMode)' == 'true'">rel/d$(VsTargetMajorVersion).$(MinorNuGetVersion)</VsTargetBranch>
<VsTargetChannel Condition="'$(IsEscrowMode)' == 'true'">int.d$(VsTargetMajorVersion).$(MinorNuGetVersion)</VsTargetChannel>
<!-- Specifies the SDK version to download to use for testing. Ideally, this is the same .NET SDK version NuGet inserts into.
The first value represents the channel, the second value represents the exact SDK version to be download. If a version is not specified, the latest version from the channel will be downloaded.
Note that multiple SDKs can be downloaded by using `;` as a separator.
The channel needs to be two-part version in A.B format, or three-part version in A.B.Cxx format.
The version needs to be the exact version number if specified.
e.g. 5.0;6.0 means install the latest versions from both channel 5.0 and channel 6.0.
6.0:6.0.100-preview.7.21379.14 means install the preview version 6.0.100-preview.7.21379.14.
Refer to https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script#options for more details.
-->
<CliBranchForTesting Condition="'$(CliBranchForTesting)' == ''">7.0</CliBranchForTesting>
</PropertyGroup>

<!-- Config -->
Expand Down Expand Up @@ -90,7 +80,4 @@
<Target Name="GetVsTargetChannelForTests">
<Message Text="$(VsTargetChannelForTests)" Importance="High"/>
</Target>
<Target Name="GetCliBranchForTesting">
<Message Text="$(CliBranchForTesting)" Importance="High"/>
</Target>
</Project>
14 changes: 13 additions & 1 deletion configure.cmd
Original file line number Diff line number Diff line change
@@ -1,2 +1,14 @@
@echo off
powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -Command "%~dpn0.ps1" %*
powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -Command "%~dpn0.ps1" -SkipDotnetInfo %*
IF ERRORLEVEL 1 (
EXIT /B %ERRORLEVEL%
)

SET "DOTNET_ROOT=%~dp0cli"
SET DOTNET_MULTILEVEL_LOOKUP=0
SET "PATH=%~dp0cli;%PATH%"
dotnet --info
IF ERRORLEVEL 1 (
ECHO "dotnet --info exited with non-zero exit code" 1>&2
EXIT /B 1
)
5 changes: 3 additions & 2 deletions configure.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ Param (
[switch]$CleanCache,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any chance while you're at it, you could move the strong name key disable code out of build.ps1 and into configure.ps1? It's something that needs to be configured once per machine, not every build.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I wasn't sure if it needed to be configured unless you want to debug our VSIX? Or is it safe to disable it on everyone's machine that runs this script?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it needs to be disabled not only to run the VSIX, but also to run the .NET Framework restore task, the non-ilmerged nuget.exe (maybe even the ilmerged nuget.exe). I'm not sure about running tests. Maybe it's needed for that too. Basically, I think strong name validation for our 2 keys should be disabled for everyone running the script on Windows.

[Alias('f')]
[switch]$Force,
[switch]$RunTest
[switch]$RunTest,
[switch]$SkipDotnetInfo
)

$ErrorActionPreference = 'Stop'
Expand All @@ -42,7 +43,7 @@ Invoke-BuildStep 'Configuring git repo' {
} -ev +BuildErrors

Invoke-BuildStep 'Installing .NET CLI' {
Install-DotnetCLI -Force:$Force
Install-DotnetCLI -Force:$Force -SkipDotnetInfo:$SkipDotnetInfo
} -ev +BuildErrors

# Restoring tools required for build
Expand Down
82 changes: 82 additions & 0 deletions configure.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
#!/usr/bin/env bash

CLI_DIR="$(pwd)/cli"

# Download the CLI install script to cli
echo "Installing .NET SDKs..."
mkdir -p $CLI_DIR
curl -o $CLI_DIR/dotnet-install.sh -L https://dot.net/v1/dotnet-install.sh --silent
if (( $? )); then
echo "Could not download 'dotnet-install.sh' script. Please check your network and try again!"
return 1
fi

# Run install.sh for cli
chmod +x $CLI_DIR/dotnet-install.sh

# If the DOTNET_SDK_VERSIONS environment variable is set, use its value instead of the ones in DotNetSdkVersions.txt
if [ "$DOTNET_SDK_VERSIONS" != "" ]; then
echo "Using environment variable DOTNET_SDK_VERSIONS instead of DotNetSdkVersions.txt. Value: '$DOTNET_SDK_VERSIONS'"
IFS=';' read -ra array <<< "$DOTNET_SDK_VERSIONS"
for CliArgs in "${array[@]}";
do
echo "'cli/dotnet-install.sh -InstallDir $CLI_DIR -NoPath $CliArgs'"

cli/dotnet-install.sh -InstallDir $CLI_DIR -NoPath $CliArgs
if (( $? )); then
echo "The .NET install failed!"
return 1
fi
done
else
# Get CLI Branches for testing
cat build/DotNetSdkVersions.txt | while IFS=$'\r' read -r CliArgs || [[ -n $line ]];
do
if [ "${CliArgs:0:1}" != "#" ] || [ "$CliArgs" == "" ]; then
echo "'cli/dotnet-install.sh -InstallDir $CLI_DIR -NoPath $CliArgs'"

cli/dotnet-install.sh -InstallDir $CLI_DIR -NoPath $CliArgs
if (( $? )); then
echo "The .NET install failed!"
return 1
fi
fi
done
fi

export DOTNET_ROOT="$CLI_DIR"
export DOTNET_MULTILEVEL_LOOKUP="0"
export "PATH=$CLI_DIR:$PATH"

if [ "$CI" == "true" ]; then
echo "##vso[task.setvariable variable=DOTNET_ROOT;isOutput=false;issecret=false;]$CLI_DIR"
echo "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP;isOutput=false;issecret=false;]0"
echo "##vso[task.prependpath]$CLI_DIR"
fi

# Display .NET CLI info
dotnet --info
if [ $? -ne 0 ]; then
echo "dotnet is not available on the PATH!"
return 1
fi

echo "=================================================================="

echo "Initializing submodules..."
git submodule init
git submodule update

echo "Restoring bootstrap packages..."
dotnet msbuild build/bootstrap.proj /Target:Restore
if [ $? -ne 0 ]; then
echo "Bootstrap failed!!"
return 1
fi

echo "Restoring NuGet packages..."
dotnet msbuild build/build.proj /Target:Restore "/ConsoleLoggerParameters:Verbosity=Minimal;Summary;ForceNoAlign" /MaxCPUCount /NodeReuse:false
if [ $? -ne 0 ]; then
echo "Restore packages failed!!"
return 1
fi
7 changes: 0 additions & 7 deletions eng/pipelines/templates/CrossFramework_Tests_On_Windows.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,4 @@
steps:
- task: UseDotNet@2
displayName: 'Use .NET Core sdk'
inputs:
packageType: sdk
version: 7.x
installationPath: $(Agent.TempDirectory)/dotnet

- task: PowerShell@2
displayName: "Print Environment Variables"
inputs:
Expand Down
Loading