Allow users to supply their own SDK for bootstrapping (#1455)

build-source-tarball.sh

@@ -10,6 +10,7 @@ usage() {
     echo "  --skip-prebuilt-check              do not confirm that all prebuilt packages used are either reference packages, previously-built, or known extras"
     echo "  --with-ref-packages <dir>          use the specified directory of available reference packages to determine what prebuilts to delete, instead of downloading the most recent version"
     echo "  --with-packages <dir>              use the specified directory of available previously-built packages to determine what prebuilts to delete, instead of downloading the most recent version"
+    echo "  --with-sdk <dir>                   use the specified SDK to check out source code.  do not copy it to the tarball.  an external SDK will be required to build from the tarball."
     echo "use -- to send the remaining arguments to build.sh"
 }
 
@@ -48,6 +49,7 @@ TARBALL_ROOT=$1
 shift
 
 SKIP_BUILD=0
+CUSTOM_SDK_DIR=''
 INCLUDE_LEAK_DETECTION=0
 MINIMIZE_DISK_USAGE=0
 SKIP_PREBUILT_ENFORCEMENT=0
@@ -75,6 +77,16 @@ while :; do
         --skip-prebuilt-check)
             SKIP_PREBUILT_ENFORCEMENT=1
             ;;
+        --with-sdk)
+            CUSTOM_SDK_DIR="$2"
+            if [ ! -d "$CUSTOM_SDK_DIR" ]; then
+                echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist"
+            fi
+            if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then
+                echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' not found or not executable"
+            fi
+            shift
+            ;;
         --with-ref-packages)
             CUSTOM_REF_PACKAGES_DIR="$2"
             if [ ! -d "$CUSTOM_REF_PACKAGES_DIR" ]; then
@@ -129,13 +141,20 @@ if [ -e "$TARBALL_ROOT" ]; then
 fi
 
 export SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"
-sdkLine=`grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json"`
-sdkPattern="\"dotnet\" *: *\"(.*)\""
-if [[ $sdkLine =~ $sdkPattern ]]; then
-  export SDK_VERSION=${BASH_REMATCH[1]}
+if [ -d "$CUSTOM_SDK_DIR" ]; then
+  export SDK_VERSION=`"$CUSTOM_SDK_DIR/dotnet" --version`
+  echo "Using custom bootstrap SDK from '$CUSTOM_SDK_DIR', version $SDK_VERSION"
+  CLI_PATH="$CUSTOM_SDK_DIR"
+else
+  sdkLine=`grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json"`
+  sdkPattern="\"dotnet\" *: *\"(.*)\""
+  if [[ $sdkLine =~ $sdkPattern ]]; then
+    export SDK_VERSION=${BASH_REMATCH[1]}
+    CLI_DIR=".dotnet"
+    CLI_PATH="$SCRIPT_ROOT/$CLI_DIR"
+  fi
+  echo "Found bootstrap SDK $SDK_VERSION"
 fi
-echo "Found bootstrap SDK $SDK_VERSION"
-CLI_PATH="$SCRIPT_ROOT/.dotnet"
 DarcVersion=$(cat $SCRIPT_ROOT/DarcVersion.txt)
 DARC_DLL="$CLI_PATH/tools/.store/microsoft.dotnet.darc/$DarcVersion/microsoft.dotnet.darc/$DarcVersion/tools/netcoreapp3.0/any/Microsoft.DotNet.Darc.dll"
 
@@ -253,7 +272,11 @@ cp $SCRIPT_ROOT/global.json $TARBALL_ROOT/
 cp $SCRIPT_ROOT/DarcVersion.txt $TARBALL_ROOT/
 cp $SCRIPT_ROOT/ProdConFeed.txt $TARBALL_ROOT/
 cp $SCRIPT_ROOT/smoke-test* $TARBALL_ROOT/
-cp -r $CLI_PATH $TARBALL_ROOT/
+if [ ! -d "$CUSTOM_SDK_DIR" ]; then
+  cp -r $CLI_PATH $TARBALL_ROOT/
+  rm -rf $TARBALL_ROOT/$CLI_DIR/shared/2.1.0/
+  rm -rf $TARBALL_ROOT/$CLI_DIR/tools/
+fi
 cp -r $SCRIPT_ROOT/eng $TARBALL_ROOT/
 cp -r $SCRIPT_ROOT/keys $TARBALL_ROOT/
 cp -r $SCRIPT_ROOT/patches $TARBALL_ROOT/
@@ -263,8 +286,6 @@ cp -r $SCRIPT_ROOT/tools-local $TARBALL_ROOT/
 rm -rf $TARBALL_ROOT/tools-local/arcade-services/
 rm -rf $TARBALL_ROOT/tools-local/tasks/*/bin
 rm -rf $TARBALL_ROOT/tools-local/tasks/*/obj
-rm -rf $TARBALL_ROOT/.dotnet/shared/2.1.0/
-rm -rf $TARBALL_ROOT/.dotnet/tools/
 cp -r $SCRIPT_ROOT/bin/git-info $TARBALL_ROOT/
 
 cp $SCRIPT_ROOT/support/tarball/build.sh $TARBALL_ROOT/build.sh

build.sh

@@ -2,6 +2,18 @@
 set -euo pipefail
 IFS=$'\n\t'
 
+usage() {
+    echo "usage: $0 [options]"
+    echo "options:"
+    echo "  -test                          run supported individal repo unit tests"
+    echo "  --run-smoke-test               run smoke tests"
+    echo "  --publish-prebuilt-report      publish prebuilt report data to Azure Storage"
+    echo "  --generate-prebuilt-data       generate prebuilt burndown data"
+    echo "  --with-sdk <dir>               use the SDK in the specified directory for bootstrapping"
+    echo "extra arguments will be passed to MSBuild"
+    echo ""
+}
+
 # resolve $SOURCE until the file is no longer a symlink
 source="${BASH_SOURCE[0]}"
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
@@ -14,18 +26,6 @@ while [[ -h $source ]]; do
   [[ $source != /* ]] && source="$scriptroot/$source"
 done
 
-arcadeLine=`grep -m 1 'Microsoft\.DotNet\.Arcade\.Sdk' "$scriptroot/global.json"`
-sdkLine=`grep -m 1 'dotnet' "$scriptroot/global.json"`
-arcadePattern="\"Microsoft\.DotNet\.Arcade\.Sdk\" *: *\"(.*)\""
-sdkPattern="\"dotnet\" *: *\"(.*)\""
-if [[ $arcadeLine =~ $arcadePattern ]]; then
-  export ARCADE_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
-fi
-if [[ $sdkLine =~ $sdkPattern ]]; then
-  export SDK_VERSION=${BASH_REMATCH[1]}
-fi
-echo "Found bootstrap SDK $SDK_VERSION, bootstrap Arcade $ARCADE_BOOTSTRAP_VERSION"
-
 if [ -z "${HOME:-}" ]; then
     export HOME="$scriptroot/.home"
     mkdir "$HOME"
@@ -36,6 +36,7 @@ if grep -q '\(/docker/\|/docker-\)' "/proc/1/cgroup"; then
 fi
 
 alternateTarget=false
+CUSTOM_SDK_DIR=''
 
 for arg do
   shift
@@ -53,10 +54,45 @@ for arg do
     (--generate-prebuilt-data) set -- "$@" "/t:GeneratePrebuiltBurndownData"
             alternateTarget=true
             ;;
+    (--with-sdk)
+            CUSTOM_SDK_DIR="$(cd -P "$1" && pwd)"
+            if [ ! -d "$CUSTOM_SDK_DIR" ]; then
+                echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist"
+                exit 1
+            fi
+            if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then
+                echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' does not exist or is not executable"
+                exit 1
+            fi
+            shift
+            ;;
        (*) set -- "$@" "$arg" ;;
   esac
 done
 
+arcadeLine=`grep -m 1 'Microsoft\.DotNet\.Arcade\.Sdk' "$scriptroot/global.json"`
+sdkLine=`grep -m 1 'dotnet' "$scriptroot/global.json"`
+arcadePattern="\"Microsoft\.DotNet\.Arcade\.Sdk\" *: *\"(.*)\""
+sdkPattern="\"dotnet\" *: *\"(.*)\""
+if [[ $arcadeLine =~ $arcadePattern ]]; then
+  export ARCADE_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
+fi
+if [ -d "$CUSTOM_SDK_DIR" ]; then
+  export SDK_VERSION=`"$CUSTOM_SDK_DIR/dotnet" --version`
+  echo "Using custom bootstrap SDK from '$CUSTOM_SDK_DIR', version $SDK_VERSION"
+  CLIPATH="$CUSTOM_SDK_DIR"
+  SDKPATH="$CLIPATH/sdk/$SDK_VERSION"
+  export _InitializeDotNetCli="$CLIPATH"
+  export CustomDotNetSdkDir="$CLIPATH"
+  else
+  if [[ $sdkLine =~ $sdkPattern ]]; then
+    export SDK_VERSION=${BASH_REMATCH[1]}
+    CLIPATH="$scriptroot/.dotnet"
+    SDKPATH="$CLIPATH/sdk/$SDK_VERSION"
+  fi
+fi
+echo "Found bootstrap SDK $SDK_VERSION, bootstrap Arcade $ARCADE_BOOTSTRAP_VERSION"
+
 if [ "$alternateTarget" == "false" ] && [[ "${SOURCE_BUILD_SKIP_SUBMODULE_CHECK:-default}" == "default" || $SOURCE_BUILD_SKIP_SUBMODULE_CHECK == "0" || $SOURCE_BUILD_SKIP_SUBMODULE_CHECK == "false" ]]; then
   source "$scriptroot/check-submodules.sh"
 fi
@@ -70,9 +106,6 @@ set -x
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
 if [ "$alternateTarget" == "true" ]; then
-  CLIPATH="$scriptroot/.dotnet"
-  SDKPATH="$CLIPATH/sdk/$SDK_VERSION"
-
   "$CLIPATH/dotnet" $SDKPATH/MSBuild.dll "$scriptroot/build.proj" /bl:source-build-test.binlog /flp:v=diag /clp:v=m "$@"
 else
   "$scriptroot/eng/common/build.sh" --restore --build -c Release --warnaserror false -bl /flp:v=diag $@ /p:Projects="$scriptroot/build.proj"

dir.props

@@ -21,6 +21,7 @@
     <TargetsDir>$(ProjectDir)targets/</TargetsDir>
     <KeysDir>$(ProjectDir)keys/</KeysDir>
     <DotNetCliToolDir>$(ProjectDir).dotnet/</DotNetCliToolDir>
+    <DotNetCliToolDir Condition="'$(CustomDotNetSdkDir)' != ''">$([MSBuild]::EnsureTrailingSlash('$(CustomDotNetSdkDir)'))</DotNetCliToolDir>
     <DotnetToolCommand>$(DotNetCliToolDir)dotnet</DotnetToolCommand>
     <PatchesDir>$(ProjectDir)patches/</PatchesDir>
     <PackagesDir>$(ProjectDir)packages/restored/</PackagesDir>

support/tarball/build.sh

@@ -7,6 +7,7 @@ usage() {
     echo "options:"
     echo "  --with-ref-packages <dir>          use the specified directory of reference packages"
     echo "  --with-packages <dir>              use the specified directory of previously-built packages"
+    echo "  --with-sdk <dir>                   use the SDK in the specified directory for bootstrapping"
     echo "use -- to send the remaining arguments to MSBuild"
     echo ""
 }
@@ -16,6 +17,7 @@ SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"
 MSBUILD_ARGUMENTS=("/p:OfflineBuild=true" "/flp:v=detailed")
 CUSTOM_REF_PACKAGES_DIR=''
 CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR=''
+CUSTOM_SDK_DIR=''
 
 while :; do
     if [ $# -le 0 ]; then
@@ -42,6 +44,18 @@ while :; do
             MSBUILD_ARGUMENTS+=( "/p:CustomPrebuiltSourceBuiltPackagesPath=$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" )
             shift
             ;;
+        --with-sdk)
+            CUSTOM_SDK_DIR="$(cd -P "$2" && pwd)"
+            if [ ! -d "$CUSTOM_SDK_DIR" ]; then
+                echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist"
+                exit 1
+            fi
+            if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then
+                echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' does not exist or is not executable"
+                exit 1
+            fi
+            shift
+            ;;
         --)
             shift
             echo "Detected '--': passing remaining parameters '$@' as build.sh arguments."
@@ -60,10 +74,19 @@ while :; do
     shift
 done
 
-sdkLine=`grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json"`
-sdkPattern="\"dotnet\" *: *\"(.*)\""
-if [[ $sdkLine =~ $sdkPattern ]]; then
-  export SDK_VERSION=${BASH_REMATCH[1]}
+if [ -d "$CUSTOM_SDK_DIR" ]; then
+  export SDK_VERSION=`"$CUSTOM_SDK_DIR/dotnet" --version`
+  export CLI_ROOT="$CUSTOM_SDK_DIR"
+  export _InitializeDotNetCli="$CLI_ROOT/dotnet"
+  export CustomDotNetSdkDir="$CLI_ROOT"
+  echo "Using custom bootstrap SDK from '$CLI_ROOT', version '$SDK_VERSION'"
+else
+  sdkLine=`grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json"`
+  sdkPattern="\"dotnet\" *: *\"(.*)\""
+  if [[ $sdkLine =~ $sdkPattern ]]; then
+    export SDK_VERSION=${BASH_REMATCH[1]}
+    export CLI_ROOT="$SCRIPT_ROOT/.dotnet"
+  fi
 fi
 
 packageVersionsPath=''
@@ -106,7 +129,6 @@ if [[ $dotNetHostLine =~ $versionPattern ]]; then
 fi
 
 echo "Found bootstrap SDK $SDK_VERSION, bootstrap Arcade $ARCADE_BOOTSTRAP_VERSION, bootstrap SourceLink $SOURCE_LINK_BOOTSTRAP_VERSION, bootstrap DotNetHost $DOTNET_HOST_BOOTSTRAP_VERSION"
-CLI_ROOT="$SCRIPT_ROOT/.dotnet"
 
 export DOTNET_CLI_TELEMETRY_OPTOUT=1
 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1