Install darc rather than building it via arcade-services (#1315)

* directly call darc-init * add runtime to support darc, add tools-path, remove un-needed projects * Use the latest darc version * remove darc and dependent stuff from tarball * remove extra stuff from tarball * add environment ld_path to build tarball * add darcVersion to tarball
dotnet · Oct 21, 2019 · fae4fa8 · fae4fa8
1 parent 8cbaf0a
commit fae4fa8
Show file tree

Hide file tree

Showing 8 changed files with 48 additions and 41 deletions.
diff --git a/DarcVersion.txt b/DarcVersion.txt
@@ -0,0 +1 @@
+1.1.0-beta.19515.3
diff --git a/build-source-tarball.sh b/build-source-tarball.sh
@@ -79,7 +79,8 @@ if [[ $sdkLine =~ $sdkPattern ]]; then
 fi
 echo "Found bootstrap SDK $SDK_VERSION"
 CLI_PATH="$SCRIPT_ROOT/.dotnet"
-DARC_DLL="$SCRIPT_ROOT/tools-local/arcade-services/artifacts/bin/Microsoft.DotNet.Darc/Release/netcoreapp3.0/Microsoft.DotNet.Darc.dll"
+DarcVersion=$(cat $SCRIPT_ROOT/DarcVersion.txt)
+DARC_DLL="$CLI_PATH/tools/.store/microsoft.dotnet.darc/$DarcVersion/microsoft.dotnet.darc/$DarcVersion/tools/netcoreapp2.1/any/Microsoft.DotNet.Darc.dll"
 
 if [ $SKIP_BUILD -ne 1 ]; then
 
@@ -146,6 +147,10 @@ git submodule foreach --quiet --recursive '
 
 # Now re-uberclone into the tarball src directory.  Since we reuse the .gitdirs, this shouldn't hit the network at all.
 ignored_repos="https://dev.azure.com/dnceng/internal/_git/dotnet-optimization;https://dev.azure.com/devdiv/DevDiv/_git/DotNet-Trusted;https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Trusted;https://dnceng@dev.azure.com/dnceng/internal/_git/dotnet-optimization;https://dev.azure.com/dnceng/internal/_git/dotnet-core-setup;https://github.com/dotnet/source-build-reference-packages"
+
+#export the LC_LIB_PATH for libgit2 so file as fedora fails to find it in the repodir
+export LD_LIBRARY_PATH=$CLI_PATH/tools/.store/microsoft.dotnet.darc/$DarcVersion/microsoft.dotnet.darc/$DarcVersion/tools/netcoreapp2.1/any/runtimes/rhel-x64/native/
+
 "$CLI_PATH/dotnet" "$DARC_DLL" clone --repos-folder=$TARBALL_ROOT/src/ --git-dir-folder $SCRIPT_ROOT/.git/modules/src/ --include-toolset --ignore-repos "$ignored_repos" --azdev-pat bogus --github-pat bogus --depth 0 --debug
 
 # now we don't need .git/modules/src or Darc anymore
@@ -182,6 +187,7 @@ cp $SCRIPT_ROOT/*.proj $TARBALL_ROOT/
 cp $SCRIPT_ROOT/*.props $TARBALL_ROOT/
 cp $SCRIPT_ROOT/*.targets $TARBALL_ROOT/
 cp $SCRIPT_ROOT/global.json $TARBALL_ROOT/
+cp $SCRIPT_ROOT/DarcVersion.txt $TARBALL_ROOT/
 cp $SCRIPT_ROOT/ProdConFeed.txt $TARBALL_ROOT/
 cp $SCRIPT_ROOT/smoke-test* $TARBALL_ROOT/
 cp -r $CLI_PATH $TARBALL_ROOT/
@@ -192,6 +198,9 @@ cp -r $SCRIPT_ROOT/scripts $TARBALL_ROOT/
 cp -r $SCRIPT_ROOT/repos $TARBALL_ROOT/
 cp -r $SCRIPT_ROOT/tools-local $TARBALL_ROOT/
 rm -rf $TARBALL_ROOT/tools-local/arcade-services/
+rm -rf $TARBALL_ROOT/.dotnet/shared/2.1.0/
+rm -rf $TARBALL_ROOT/.dotnet/tools/
+rm -rf $TARBALL_ROOT/.dotnet/host/fxr/2.1.0/
 cp -r $SCRIPT_ROOT/bin/git-info $TARBALL_ROOT/
 
 cp $SCRIPT_ROOT/support/tarball/build.sh $TARBALL_ROOT/build.sh

diff --git a/build.sh b/build.sh
@@ -69,6 +69,9 @@ export NUGET_PACKAGES="$scriptroot/packages/restored/"
 set -x
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
+# runtime 2.1.0 required for darc
+"$scriptroot/eng/common/dotnet-install.sh"  -runtime dotnet -version 2.1.0
+
 if [ "$alternateTarget" == "true" ]; then
   CLIPATH="$scriptroot/.dotnet"
   SDKPATH="$CLIPATH/sdk/$SDK_VERSION"

diff --git a/dir.props b/dir.props
@@ -58,7 +58,9 @@
     <ToolPackageExtractDir>$(ToolsDir)source-built/</ToolPackageExtractDir>
     <ToolsLocalDir>$(ProjectDir)tools-local/</ToolsLocalDir>
     <TaskDirectory>$(ToolsLocalDir)tasks/</TaskDirectory>
-    <DarcDll>$(ToolsLocalDir)arcade-services/artifacts/bin/Microsoft.DotNet.Darc/Release/netcoreapp3.0/Microsoft.DotNet.Darc.dll</DarcDll>
+    <DarcInstallDir>$(DotNetCliToolDir)tools/</DarcInstallDir>
+    <DarcVersion>$([System.IO.File]::ReadAllText('$(ProjectDir)DarcVersion.txt').Trim())</DarcVersion>
+    <DarcDll>$(DotNetCliToolDir)tools/.store/microsoft.dotnet.darc/$(DarcVersion)/microsoft.dotnet.darc/$(DarcVersion)/tools/netcoreapp2.1/any/Microsoft.DotNet.Darc.dll</DarcDll>
     <TasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks/bin/Debug/netstandard2.0/</TasksBinDir>
     <SourceBuildTasksAssembly>$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll</SourceBuildTasksAssembly>
     <XPlatTasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks.XPlat/bin/Debug/netstandard2.0/</XPlatTasksBinDir>

diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
@@ -3,6 +3,7 @@
 source="${BASH_SOURCE[0]}"
 darcVersion=''
 versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16"
+verbosity=m
 
 while [[ $# > 0 ]]; do
   opt="$(echo "$1" | awk '{print tolower($0)}')"
@@ -15,6 +16,14 @@ while [[ $# > 0 ]]; do
       versionEndpoint=$2
       shift
       ;;
+    --verbosity)
+      verbosity=$2
+      shift
+      ;;
+    --toolpath)
+      toolpath=$2
+      shift
+      ;;
     *)
       echo "Invalid argument: $1"
       usage
@@ -34,7 +43,6 @@ while [[ -h "$source" ]]; do
   [[ $source != /* ]] && source="$scriptroot/$source"
 done
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-verbosity=m
 
 . "$scriptroot/tools.sh"
 
@@ -48,17 +56,27 @@ function InstallDarcCli {
   InitializeDotNetCli
   local dotnet_root=$_InitializeDotNetCli
 
-  local uninstall_command=`$dotnet_root/dotnet tool uninstall $darc_cli_package_name -g`
-  local tool_list=$($dotnet_root/dotnet tool list -g)
-  if [[ $tool_list = *$darc_cli_package_name* ]]; then
-    echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
+  if [ -z "$toolpath" ]; then
+    local tool_list=$($dotnet_root/dotnet tool list -g)
+    if [[ $tool_list = *$darc_cli_package_name* ]]; then
+      echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
+    fi
+  else
+    local tool_list=$($dotnet_root/dotnet tool list --tool-path "$toolpath")
+    if [[ $tool_list = *$darc_cli_package_name* ]]; then
+      echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name --tool-path "$toolpath")
+    fi
   fi
 
-  local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"
+  local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"
 
   echo "Installing Darc CLI version $darcVersion..."
   echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
-  echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
+  if [ -z "$toolpath" ]; then
+    echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
+  else
+    echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath")
+  fi
 }
 
 InstallDarcCli
diff --git a/tools-local/ILAsmVersion.txt b/tools-local/ILAsmVersion.txt
diff --git a/tools-local/ildasm.depproj b/tools-local/ildasm.depproj
diff --git a/tools-local/init-build.proj b/tools-local/init-build.proj
@@ -23,7 +23,7 @@
     Inputs="$(TargetInfoProps)"
     Outputs="$(BuildCompetedSuccessSemaphore)"
     >
-    <CallTarget Targets="UnpackTarballs;BuildXPlatTasks;BuildDarc;FetchSources;WriteSourceLinkMetadata;BuildTasks;ExtractToolPackage;InstallSourceBuildSdkResolver;WriteDynamicPropsToStaticPropsFiles;GenerateRootFs;ApplyPatches;PoisonPrebuiltPackages" />
+    <CallTarget Targets="UnpackTarballs;BuildXPlatTasks;AcquireDarc;FetchSources;WriteSourceLinkMetadata;BuildTasks;ExtractToolPackage;InstallSourceBuildSdkResolver;WriteDynamicPropsToStaticPropsFiles;GenerateRootFs;ApplyPatches;PoisonPrebuiltPackages" />
     <Touch Files="$(BuildCompetedSuccessSemaphore)" AlwaysCreate="true" />
   </Target>
 
@@ -153,13 +153,13 @@
     <MarkAndCatalogPackages PackagesToMark="@(PrebuiltSourceBuiltPackages)" CatalogOutputFilePath="$(SourceBuiltPoisonReportDataFile)" MarkerFileName="$(SourceBuiltPoisonMarkerFile)" />
   </Target>
 
-  <Target Name="BuildDarc" Condition="'$(OfflineBuild)' != 'true'">
+  <Target Name="AcquireDarc" Condition="'$(OfflineBuild)' != 'true'">
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Building Darc." />
-    <Exec Command="$(DotNetCliToolDir)dotnet build -c Release $(MSBuildProjectDirectory)/arcade-services/src/Microsoft.DotNet.Darc/src/Darc/Microsoft.DotNet.Darc.csproj /p:TargetFramework=netcoreapp3.0" />
+    <Exec Command="$(ProjectDir)eng/common/darc-init.sh --darcversion $(DarcVersion) --toolpath $(DarcInstallDir)" />
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done building Darc." />
   </Target>
 
-  <Target Name="FetchSources" Condition="'$(OfflineBuild)' != 'true'" DependsOnTargets="BuildDarc">
+  <Target Name="FetchSources" Condition="'$(OfflineBuild)' != 'true'" DependsOnTargets="AcquireDarc">
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Getting source code for repos in Version.Details.xml." />
     <PropertyGroup>
       <DarcCloneArguments>clone</DarcCloneArguments>
@@ -183,11 +183,11 @@
       <DarcCloneCommand>$(DotNetCliToolDir)dotnet $(DarcDll) $(DarcCloneArguments)</DarcCloneCommand>
     </PropertyGroup>
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Running Darc command: '$(DarcCloneCommand)' from working dir $(ProjectDir)" />
-    <Exec Command="$(DarcCloneCommand)" WorkingDirectory="$(ProjectDir)" />
+    <Exec Command="$(DarcCloneCommand)" WorkingDirectory="$(ProjectDir)" EnvironmentVariables="LD_LIBRARY_PATH=$(DotNetCliToolDir)/tools/.store/microsoft.dotnet.darc/$(DarcVersion)/microsoft.dotnet.darc/$(DarcVersion)/tools/netcoreapp2.1/any/runtimes/rhel-x64/native/" />
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done getting source code for repos in Version.Details.xml." />
   </Target>
 
-  <Target Name="WriteSourceLinkMetadata" Condition="'$(OfflineBuild)' != 'true'" DependsOnTargets="BuildDarc;FetchSources">
+  <Target Name="WriteSourceLinkMetadata" Condition="'$(OfflineBuild)' != 'true'" DependsOnTargets="AcquireDarc;FetchSources">
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Writing out SourceLink metadata." />
     <WriteSourceRepoProperties
        VersionDetailsFile="$(ProjectDir)eng/Version.Details.xml"