dotnet · dseefeld · Feb 24, 2020 · Jan 16, 2020 · Jan 16, 2020 · Jan 17, 2020
diff --git a/.vsts.pipelines/jobs/ci-linux.yml b/.vsts.pipelines/jobs/ci-linux.yml
@@ -14,7 +14,7 @@ jobs:
   strategy:
     matrix: ${{ parameters.matrix }}
   pool: ${{ parameters.pool }}
-  timeoutInMinutes: 270
+  timeoutInMinutes: 300
   variables:
     # Prefix to distinguish artifacts from different legs.
     artifactName: ${{ format('{0} $(type)', parameters.job) }}
@@ -72,7 +72,7 @@ jobs:
         /p:AzDoPat=$(System.AccessToken)
       du -h $(rootDirectory) | sort -h | tail -n 50
     displayName: Build source-build
-    timeoutInMinutes: 150
+    timeoutInMinutes: 180
 
   # Generate prebuilt burndown data
   - script: |
@@ -106,8 +106,8 @@ jobs:
       du -h $(rootDirectory) | sort -h | tail -n 50
     displayName: Run unit tests
     condition: and(succeeded(), eq(variables['runUnitTests'], true))
-    timeoutInMinutes: 80
-    continueOnError: true
+    timeoutInMinutes: 120
+    continueOnError: false
 
   # Create tarball.
   - script: |
@@ -201,14 +201,14 @@ jobs:
       if [ "$(reportPrebuiltLeaks)" = "true" ]; then
         poisonArg="/p:EnablePoison=true"
       fi
-      $(docker.run) $(docker.tb.map) $(docker.tb.work) $networkArg $(imageName) "$(tarballName)/build.sh" \
+      $(docker.run) $(docker.tb.map) $(docker.tb.work) $networkArg $(imageName) "$(tarballName)/build.sh" -- \
         /p:Configuration=$(sb.configuration) \
         /p:PortableBuild=$(sb.portable) \
         /p:FailOnPrebuiltBaselineError=true \
         $poisonArg
       du -h $(rootDirectory) | sort -h | tail -n 50
     displayName: Build tarball
-    timeoutInMinutes: 120
+    timeoutInMinutes: 150
     condition: and(succeeded(), eq(variables['sb.tarball'], true))
 
   # Run smoke tests.

diff --git a/.vsts.pipelines/jobs/ci-linux_bootstrap.yml b/.vsts.pipelines/jobs/ci-linux_bootstrap.yml
@@ -35,21 +35,22 @@ jobs:
     type: Production
 
   steps:
+  - template: ../steps/cleanup-unneeded-files.yml
   - template: ../steps/docker-cleanup-linux.yml
 
   - task: DownloadBuildArtifacts@0
     displayName: Download CentOS artifacts
     inputs:
       buildType: 'current'
-      itemPattern: 'Tarball centos71 Offline/$(tarballName).tar.gz'
+      itemPattern: 'Tarball centos71 Offline/$(tarballName)*.tar.gz'
       artifactName: 'Tarball centos71 Offline'
       downloadPath: '$(System.ArtifactsDirectory)/boostrap_root'
 
   - script: |
       set -ex
       df -h
       $(docker.run) $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
-      git clone https://github.com/dotnet/source-build-reference-packages'
+      git clone --depth 1 https://github.com/dotnet/source-build-reference-packages'
       du -h $(rootDirectory) | sort -h | tail -n 50
     displayName: clone source-build-reference-packages
     condition: always()

diff --git a/.vsts.pipelines/steps/run-bootstrap.yml b/.vsts.pipelines/steps/run-bootstrap.yml
@@ -1,15 +1,14 @@
-# Delete some files from the default image since we're tight on disk space.
-# List is from https://github.com/microsoft/azure-pipelines-image-generation/pull/1231#issuecomment-537174621
-
 # Extract the tarball fetched from the previous Offline stage 
 steps:
 - script: |
       set -ex 
+      mount
       df -h 
       $(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
       mkdir -p /bst/sb/tarball
-      tar -xvf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz" -C /bst/sb/tarball --no-same-owner --strip-components=1
-      rm -rf "/bst/Tarball centos71 Offline"'
+      tar -xf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz" -C /bst/sb/tarball --no-same-owner --strip-components=1
+      rm -rf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz"'
+      df -h
   displayName: Unpack Tarball
   condition: always()
 
@@ -19,6 +18,7 @@ steps:
       df -h 
       $(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
       /bst/sb/tarball/tool-bootstrapping/bootstrap.sh /bst/bootstrap_dir /bst/sb/tarball /bst/source-build-reference-packages 1'
+      df -h
   displayName: Build stage1
   condition: succeeded()
 
@@ -84,16 +84,18 @@ steps:
   displayName: Build stage1 source-build-reference-packages
   condition: succeeded()
 
-# Copy ref-packages tarball to drop directory.
+# Create and copy ref-packages tarball to drop directory.
 - script: |
     set -ex
     df -h
     $(docker.run) $(docker.bst.map) $(docker.drop.map) $(docker.bst.work) $(imageName) /bin/bash -c "
       mkdir -p /drop/tarball/reference-packages
       pushd /bst/bootstrap_dir/reference-packages
+      tar --numeric-owner -czf Private.SourceBuild.ReferencePackages.bootstrap.tar.gz *.nupkg
       find . \( \
         -iname 'Private.SourceBuild.ReferencePackages.bootstrap.tar.gz' \) \
         -exec cp {} /drop/tarball/reference-packages \;
+      rm -rf Private.SourceBuild.ReferencePackages.bootstrap.tar.gz
       popd"
     du -h $(rootDirectory) | sort -h | tail -n 50
   displayName: Copy ref-packages tarball 
@@ -105,10 +107,11 @@ steps:
       set -ex 
       df -h 
       $(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
-      rm -rf /bst/source-build-reference-packages/* 
+      rm -rf /bst/source-build-reference-packages/*
+      rm -rf /bst/source-build-reference-packages/.git
       pushd /bst/bootstrap_dir/reference-packages
-      echo "Removing everything in reference-packages except artifacts"
-      find . -type f -not -name '*SourceBuild.ReferencePackages.bootstrap.tar.gz' -delete
+      echo "Removing everything in reference-packages except nupkg artifacts"
+      find . -type f -not -name '*.nupkg' -delete
       popd'
   displayName: Clean all except tarball in source-build-reference-packages
   condition: always()
@@ -176,6 +179,17 @@ steps:
   condition: always()
   continueOnError: true
 
+- script: |
+      set -ex
+      mount
+      df -h
+      $(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
+      tar -xf "/bst/Tarball centos71 Offline/$(tarballName)-smoke-test-prereqs.tar.gz" -C /bst/bootstrap_dir/final-sdk --no-same-owner --strip-components=1
+      rm -rf "/bst/Tarball centos71 Offline/$(tarballName)-smoke-test-prereqs.tar.gz"'
+      df -h
+  displayName: Unpack smoke test prereqs
+  condition: always()
+
 - script: |
       set -ex 
       df -h 
@@ -226,4 +240,4 @@ steps:
   inputs:
     PathtoPublish: $(Build.ArtifactStagingDirectory)/drop/tarball
     ArtifactName: Tarball Artifacts $(artifactName)
-    ArtifactType: Container
+    ArtifactType: Container
diff --git a/NuGet.config b/NuGet.config
@@ -3,12 +3,11 @@
   <packageSources>
     <clear />
     <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
-    <add key="darc-pub-microsoft-msbuild-e901037-2" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-microsoft-msbuild-e901037f-2/nuget/v3/index.json" />
-    <add key="darc-pub-dotnet-templating-53a17fc-1" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-templating-53a17fce-1/nuget/v3/index.json" />
+    <add key="darc-pub-microsoft-msbuild-e901037-3" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-microsoft-msbuild-e901037f-3/nuget/v3/index.json" />
     <add key="darc-pub-dotnet-standard-a5b5f2e" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-standard-a5b5f2e1/nuget/v3/index.json" />
-    <add key="darc-pub-aspnet-Extensions-4e1be2f" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-aspnet-Extensions-4e1be2fb/nuget/v3/index.json" />
-    <add key="darc-pub-aspnet-AspNetCore-Tooling-2903677" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-aspnet-AspNetCore-Tooling-29036777/nuget/v3/index.json" />
-    <add key="darc-pub-aspnet-AspNetCore-2b7e994" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-aspnet-AspNetCore-2b7e994b/nuget/v3/index.json" />
+    <add key="darc-pub-dotnet-aspnetcore-tooling-2dab42e" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-aspnetcore-tooling-2dab42e1/nuget/v3/index.json" />
+    <add key="darc-pub-dotnet-aspnetcore-c3acdca-2" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-aspnetcore-c3acdcac-2/nuget/v3/index.json" />
+    <add key="darc-pub-aspnet-Extensions-1286a6f" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-aspnet-Extensions-1286a6ff/nuget/v3/index.json" />
     <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
     <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="nuget-build" value="https://dotnet.myget.org/F/nuget-build/api/v3/index.json" />

diff --git a/ProdConFeed.txt b/ProdConFeed.txt
@@ -1 +1 @@
-https://dotnetfeed.blob.core.windows.net/dotnet-core-3-1-ga-014709/index.json
+https://pkgs.dev.azure.com/dnceng/public/_packaging/3.1.102-servicing-014865/nuget/v3/index.json
diff --git a/build-source-tarball.sh b/build-source-tarball.sh
@@ -3,8 +3,15 @@ set -euo pipefail
 IFS=$'\n\t'
 
 usage() {
-    echo "usage: $0 <path-to-tarball-root> [--skip-build] [--enable-leak-detection] [--skip-prebuilt-check] [-- [extra build.sh args]]"
-    echo ""
+    echo "usage: $0 <path-to-tarball-root> [options]"
+    echo "options:"
+    echo "  --skip-build                       assume we have already built (requires you have built with the /p:ArchiveDownloadedPackages=true flag)"
+    echo "  --enable-leak-detection            build leaked-binary detection tasts for later use while building from this tarball"
+    echo "  --skip-prebuilt-check              do not confirm that all prebuilt packages used are either reference packages, previously-built, or known extras"
+    echo "  --with-ref-packages <dir>          use the specified directory of available reference packages to determine what prebuilts to delete, instead of downloading the most recent version"
+    echo "  --with-packages <dir>              use the specified directory of available previously-built packages to determine what prebuilts to delete, instead of downloading the most recent version"
+    echo "  --with-sdk <dir>                   use the specified SDK to check out source code.  do not copy it to the tarball.  an external SDK will be required to build from the tarball."
+    echo "use -- to send the remaining arguments to build.sh"
 }
 
 if [ -z "${1:-}" ]; then
@@ -42,9 +49,13 @@ TARBALL_ROOT=$1
 shift
 
 SKIP_BUILD=0
+CUSTOM_SDK_DIR=''
 INCLUDE_LEAK_DETECTION=0
 MINIMIZE_DISK_USAGE=0
 SKIP_PREBUILT_ENFORCEMENT=0
+CUSTOM_REF_PACKAGES_DIR=''
+CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR=''
+MAIN_BUILD_ARGS=("/p:ArchiveDownloadedPackages=true")
 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 
 while :; do
@@ -66,6 +77,34 @@ while :; do
         --skip-prebuilt-check)
             SKIP_PREBUILT_ENFORCEMENT=1
             ;;
+        --with-sdk)
+            CUSTOM_SDK_DIR="$2"
+            if [ ! -d "$CUSTOM_SDK_DIR" ]; then
+                echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist"
+            fi
+            if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then
+                echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' not found or not executable"
+            fi
+            shift
+            ;;
+        --with-ref-packages)
+            CUSTOM_REF_PACKAGES_DIR="$2"
+            if [ ! -d "$CUSTOM_REF_PACKAGES_DIR" ]; then
+                echo "Custom reference packages directory '$CUSTOM_REF_PACKAGES_DIR' does not exist"
+                exit 1
+            fi
+            MAIN_BUILD_ARGS+=( "/p:SkipDownloadingReferencePackages=true" )
+            shift
+            ;;
+        --with-packages)
+            CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR="$2"
+            if [ ! -d "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" ]; then
+                echo "Custom reference packages directory '$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR' does not exist"
+                exit 1
+            fi
+            MAIN_BUILD_ARGS+=( "/p:SkipDownloadingPreviouslySourceBuiltPackages=true" )
+            shift
+            ;;
         --)
             shift
             echo "Detected '--': passing remaining parameters '$@' as build.sh arguments."
@@ -102,13 +141,20 @@ if [ -e "$TARBALL_ROOT" ]; then
 fi
 
 export SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"
-sdkLine=`grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json"`
-sdkPattern="\"dotnet\" *: *\"(.*)\""
-if [[ $sdkLine =~ $sdkPattern ]]; then
-  export SDK_VERSION=${BASH_REMATCH[1]}
+if [ -d "$CUSTOM_SDK_DIR" ]; then
+  export SDK_VERSION=`"$CUSTOM_SDK_DIR/dotnet" --version`
+  echo "Using custom bootstrap SDK from '$CUSTOM_SDK_DIR', version $SDK_VERSION"
+  CLI_PATH="$CUSTOM_SDK_DIR"
+else
+  sdkLine=`grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json"`
+  sdkPattern="\"dotnet\" *: *\"(.*)\""
+  if [[ $sdkLine =~ $sdkPattern ]]; then
+    export SDK_VERSION=${BASH_REMATCH[1]}
+    CLI_DIR=".dotnet"
+    CLI_PATH="$SCRIPT_ROOT/$CLI_DIR"
+  fi
+  echo "Found bootstrap SDK $SDK_VERSION"
 fi
-echo "Found bootstrap SDK $SDK_VERSION"
-CLI_PATH="$SCRIPT_ROOT/.dotnet"
 DarcVersion=$(cat $SCRIPT_ROOT/DarcVersion.txt)
 DARC_DLL="$CLI_PATH/tools/.store/microsoft.dotnet.darc/$DarcVersion/microsoft.dotnet.darc/$DarcVersion/tools/netcoreapp3.0/any/Microsoft.DotNet.Darc.dll"
 
@@ -119,7 +165,7 @@ if [ $SKIP_BUILD -ne 1 ]; then
     fi
 
     $SCRIPT_ROOT/clean.sh
-    $SCRIPT_ROOT/build.sh /p:ArchiveDownloadedPackages=true "$@"
+    $SCRIPT_ROOT/build.sh  ${MAIN_BUILD_ARGS[@]} "$@"
 fi
 
 mkdir -p "$TARBALL_ROOT"
@@ -226,7 +272,11 @@ cp $SCRIPT_ROOT/global.json $TARBALL_ROOT/
 cp $SCRIPT_ROOT/DarcVersion.txt $TARBALL_ROOT/
 cp $SCRIPT_ROOT/ProdConFeed.txt $TARBALL_ROOT/
 cp $SCRIPT_ROOT/smoke-test* $TARBALL_ROOT/
-cp -r $CLI_PATH $TARBALL_ROOT/
+if [ ! -d "$CUSTOM_SDK_DIR" ]; then
+  cp -r $CLI_PATH $TARBALL_ROOT/
+  rm -rf $TARBALL_ROOT/$CLI_DIR/shared/2.1.0/
+  rm -rf $TARBALL_ROOT/$CLI_DIR/tools/
+fi
 cp -r $SCRIPT_ROOT/eng $TARBALL_ROOT/
 cp -r $SCRIPT_ROOT/keys $TARBALL_ROOT/
 cp -r $SCRIPT_ROOT/patches $TARBALL_ROOT/
@@ -236,8 +286,6 @@ cp -r $SCRIPT_ROOT/tools-local $TARBALL_ROOT/
 rm -rf $TARBALL_ROOT/tools-local/arcade-services/
 rm -rf $TARBALL_ROOT/tools-local/tasks/*/bin
 rm -rf $TARBALL_ROOT/tools-local/tasks/*/obj
-rm -rf $TARBALL_ROOT/.dotnet/shared/2.1.0/
-rm -rf $TARBALL_ROOT/.dotnet/tools/
 cp -r $SCRIPT_ROOT/bin/git-info $TARBALL_ROOT/
 
 cp $SCRIPT_ROOT/support/tarball/build.sh $TARBALL_ROOT/build.sh
@@ -255,8 +303,10 @@ cp -r $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/reference-packages/source
 cp -r $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/reference-packages/staging $TARBALL_ROOT/packages/reference/staging
 
 # Copy tarballs to ./packages/archive directory
-mkdir -p $TARBALL_ROOT/packages/archive
-cp -r $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/external-tarballs/*.tar.gz $TARBALL_ROOT/packages/archive/
+if [[ -d "$SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/external-tarballs" && ! -z "$(find $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/external-tarballs -iname '*.tar.gz')" ]]; then
+    mkdir -p $TARBALL_ROOT/packages/archive
+    cp -r $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/external-tarballs/*.tar.gz $TARBALL_ROOT/packages/archive/
+fi
 
 # Copy generated source from bin to src/generatedSrc
 cp -r $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/generatedSrc $TARBALL_ROOT/src/generatedSrc
@@ -277,10 +327,6 @@ do
     fi
 done
 
-echo 'Copying source-built packages to tarball to replace packages needed before they are built...'
-mkdir -p $TARBALL_ROOT/packages/source-built
-cp -r $SCRIPT_ROOT/Tools/source-built/coreclr-tools $TARBALL_ROOT/packages/source-built/
-
 # Setup package version props to include both source-built and running PackageVersions.props
 mkdir --parents $TARBALL_ROOT/bin/obj/$targetArchitecture/Release/
 cp $SCRIPT_ROOT/support/tarball/PackageVersions.props $TARBALL_ROOT/bin/obj/$targetArchitecture/Release/
@@ -300,8 +346,16 @@ do
     fi
 done
 
-allRefPkgs=(`tar -tf $TARBALL_ROOT/packages/archive/Private.SourceBuild.ReferencePackages.*.tar.gz | tr '[:upper:]' '[:lower:]'`)
-allSourceBuiltPkgs=(`tar -tf $TARBALL_ROOT/packages/archive/Private.SourceBuilt.Artifacts.*.tar.gz | tr '[:upper:]' '[:lower:]'`)
+if [ -d "$CUSTOM_REF_PACKAGES_DIR" ]; then
+    allRefPkgs=(`ls "$CUSTOM_REF_PACKAGES_DIR"  | tr '[:upper:]' '[:lower:]'`)
+else
+    allRefPkgs=(`tar -tf $TARBALL_ROOT/packages/archive/Private.SourceBuild.ReferencePackages.*.tar.gz | tr '[:upper:]' '[:lower:]'`)
+fi
+if [ -d "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" ]; then
+    allSourceBuiltPkgs=(`ls "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR"  | tr '[:upper:]' '[:lower:]'`)
+else
+    allSourceBuiltPkgs=(`tar -tf $TARBALL_ROOT/packages/archive/Private.SourceBuilt.Artifacts.*.tar.gz | tr '[:upper:]' '[:lower:]'`)
+fi
 
 echo 'Removing reference-packages from tarball prebuilts...'
 

diff --git a/build.proj b/build.proj
@@ -147,15 +147,15 @@
 
   <Target Name="DownloadSourceBuildReferencePackages"
           AfterTargets="Build"
-          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT'">
+          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'">
     <DownloadFile 
           SourceUrl="$(ReferencePackagesTarballUrl)$(ReferencePackagesTarballName).$(PrivateSourceBuildReferencePackagesPackageVersion).tar.gz"
           DestinationFolder="$(ExternalTarballsDir)" />
   </Target>
 
   <Target Name="DownloadSourceBuiltArtifacts"
           AfterTargets="Build"
-          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT'">
+          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingPreviouslySourceBuiltPackages)' != 'true'">
     <DownloadFile 
           SourceUrl="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltArtifactsTarballName).$(PrivateSourceBuiltArtifactsPackageVersion).tar.gz"
           DestinationFolder="$(ExternalTarballsDir)" />
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		https://dotnetfeed.blob.core.windows.net/dotnet-core-3-1-ga-014709/index.json
		https://pkgs.dev.azure.com/dnceng/public/_packaging/3.1.102-servicing-014865/nuget/v3/index.json