Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update bootstrap scripts to use params for source-build and s-b-r-p build.sh #1476

Merged
merged 4 commits into from
Feb 16, 2020
Merged

Update bootstrap scripts to use params for source-build and s-b-r-p build.sh #1476

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ae60bbf
Update bootstrap scripts to use params for source-build build.sh and
dseefeld Feb 11, 2020
f8c1473
Update CI to work with updated bootstrap script
dseefeld Feb 12, 2020
89cfda1
Fix cleanup of reference packages
dseefeld Feb 13, 2020
d2787e9
Enable smoke-test-prereqs in bootstrapping final SDK smoke tests.
dseefeld Feb 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .vsts.pipelines/jobs/ci-linux_bootstrap.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
displayName: Download CentOS artifacts
inputs:
buildType: 'current'
itemPattern: 'Tarball centos71 Offline/$(tarballName).tar.gz'
itemPattern: 'Tarball centos71 Offline/$(tarballName)*.tar.gz'
artifactName: 'Tarball centos71 Offline'
downloadPath: '$(System.ArtifactsDirectory)/boostrap_root'

Expand Down
21 changes: 17 additions & 4 deletions .vsts.pipelines/steps/run-bootstrap.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ steps:
$(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
mkdir -p /bst/sb/tarball
tar -xf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz" -C /bst/sb/tarball --no-same-owner --strip-components=1
rm -rf "/bst/Tarball centos71 Offline"'
rm -rf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz"'
df -h
displayName: Unpack Tarball
condition: always()
Expand Down Expand Up @@ -84,16 +84,18 @@ steps:
displayName: Build stage1 source-build-reference-packages
condition: succeeded()

# Copy ref-packages tarball to drop directory.
# Create and copy ref-packages tarball to drop directory.
- script: |
set -ex
df -h
$(docker.run) $(docker.bst.map) $(docker.drop.map) $(docker.bst.work) $(imageName) /bin/bash -c "
mkdir -p /drop/tarball/reference-packages
pushd /bst/bootstrap_dir/reference-packages
tar --numeric-owner -czf Private.SourceBuild.ReferencePackages.bootstrap.tar.gz *.nupkg
find . \( \
-iname 'Private.SourceBuild.ReferencePackages.bootstrap.tar.gz' \) \
-exec cp {} /drop/tarball/reference-packages \;
rm -rf Private.SourceBuild.ReferencePackages.bootstrap.tar.gz
popd"
du -h $(rootDirectory) | sort -h | tail -n 50
displayName: Copy ref-packages tarball
Expand All @@ -108,8 +110,8 @@ steps:
rm -rf /bst/source-build-reference-packages/*
rm -rf /bst/source-build-reference-packages/.git
pushd /bst/bootstrap_dir/reference-packages
echo "Removing everything in reference-packages except artifacts"
find . -type f -not -name '*SourceBuild.ReferencePackages.bootstrap.tar.gz' -delete
echo "Removing everything in reference-packages except nupkg artifacts"
find . -type f -not -name '*.nupkg' -delete
popd'
displayName: Clean all except tarball in source-build-reference-packages
condition: always()
Expand Down Expand Up @@ -177,6 +179,17 @@ steps:
condition: always()
continueOnError: true

- script: |
set -ex
mount
df -h
$(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
tar -xf "/bst/Tarball centos71 Offline/$(tarballName)-smoke-test-prereqs.tar.gz" -C /bst/bootstrap_dir/final-sdk --no-same-owner --strip-components=1
rm -rf "/bst/Tarball centos71 Offline/$(tarballName)-smoke-test-prereqs.tar.gz"'
df -h
displayName: Unpack smoke test prereqs
condition: always()

- script: |
set -ex
df -h
Expand Down
2 changes: 1 addition & 1 deletion smoke-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ restoredPackagesDir="$testingDir/packages"
testingHome="$testingDir/home"
archiveRestoredPackages=false
archivedPackagesDir="$testingDir/smoke-test-packages"
smokeTestPrebuilts="$SCRIPT_ROOT/prebuilt/smoke-test-packages"
smokeTestPrebuilts="$SCRIPT_ROOT/packages/smoke-test-packages"
runningOnline=false
runningHttps=false

Expand Down
67 changes: 15 additions & 52 deletions support/tarball/tool-bootstrapping/bootstrap.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,32 +80,16 @@ function BuildRefPkgs {
LogMessage "Copy $refPkgSourceDir to $refPkgsDir"
cp -r $refPkgSourceDir $refPkgsDir

LogMessage "Copy coreclr-tools to $refPkgsDir/source-built"
cp -r $stage1Dir/Tools/source-built/coreclr-tools/ $refPkgsDir/source-built

LogMessage "Un-tar private source-built artifacts into $refPkgsDir/source-built/"
tar -xzf $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts.*.tar.gz --directory $refPkgsDir/source-built/

LogMessage "Un-tar sdk into $refPkgsDir/.dotnet/"
if [ -d "$refPkgsDir/.dotnet/" ]; then
rm -rf $refPkgsDir/.dotnet/
LogMessage "Un-tar sdk into $stage1SdkInstallDir"
if [ -d "$stage1SdkInstallDir" ]; then
rm -rf $stage1SdkInstallDir
fi
mkdir $refPkgsDir/.dotnet/
tar -xzf $stage1Dir/bin/x64/Release/dotnet-sdk-3*.tar.gz --directory $refPkgsDir/.dotnet/

LogMessage "Update SDK version in global.json file"
sdkVer=`ls $refPkgsDir/.dotnet/sdk`
LogMessage " Sdk version = [$sdkVer]"
sed -i "s|\"dotnet\": \".*|\"dotnet\": \"$sdkVer\"|g" $refPkgsDir/global.json
mkdir $stage1SdkInstallDir
tar -xzf $stage1Dir/bin/x64/Release/dotnet-sdk-3*.tar.gz --directory $stage1SdkInstallDir

LogMessage "Build stage1 ref-pkgs"
pushd $refPkgsDir
./build.sh
popd

LogMessage "Package built reference packages into new tarball"
pushd $refPkgsDir/artifacts/reference-packages
tar --numeric-owner -czf $refPkgsDir/artifacts/Private.SourceBuild.ReferencePackages.bootstrap.tar.gz *.nupkg
./build.sh --with-packages $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts.*.tar.gz --with-sdk $stage1SdkInstallDir
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this needs to be a directory.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No. I think it's the tarball. Looking at the help text in sbrp build.sh, it says "<dir>", but it actually takes the archive, because it calls tar xf on it. https://github.com/dotnet/source-build-reference-packages/blob/9cc7dad6c4d9302c00803df5dea41c8a7a59e047/build.sh#L83

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, sorry, got this confused with the source-build version. We should maybe allow that there as well, I can log an issue.

popd

LogMessage "Completing Step2 - Build Reference Packages"
Expand Down Expand Up @@ -139,45 +123,22 @@ function buildFinalSdk {
fi

if [ "${1-default}" != "test" ]; then
LogMessage "Copy $tarballSourceDir to $finalSdkDir"
cp -r $tarballSourceDir $finalSdkDir
LogMessage "Copy $tarballSourceDir to $finalSdkDir"
cp -r $tarballSourceDir $finalSdkDir
fi

LogMessage "Remove archive packages, source-built packages and SDK from final-sdk dir"
rm -rf $finalSdkDir/packages/archive/*
rm -rf $finalSdkDir/packages/archive
rm -rf $finalSdkDir/packages/source-built/*
rm -rf $finalSdkDir/.dotnet

LogMessage "Copy stage1 coreclr-tools to $finalSdkDir/packages/source-built"
cp -r $stage1Dir/Tools/source-built/coreclr-tools $finalSdkDir/packages/source-built

LogMessage "Copy source-built packages archive from $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts*.tar.gz to $finalSdkDir/packages/archive"
cp $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts*.tar.gz $finalSdkDir/packages/archive

LogMessage "Extract .NET Core SDK from $stage1Dir/bin/x64/Release/dotnet-sdk-3*.tar.gz to $finalSdkDir/.dotnet/"
if [ -d "$finalSdkDir/.dotnet/" ]; then
rm -rf $finalSdkDir/.dotnet/
fi
mkdir $finalSdkDir/.dotnet/
tar -xzf $stage1Dir/bin/x64/Release/dotnet-sdk-3*.tar.gz --directory $finalSdkDir/.dotnet/

LogMessage "Copy reference packages from $refPkgsDir/artifacts/Private.SourceBuild.ReferencePackages.bootstrap.tar.gz to $finalSdkDir/packages/archive"
cp $refPkgsDir/artifacts/Private.SourceBuild.ReferencePackages.bootstrap.tar.gz $finalSdkDir/packages/archive

LogMessage "Update version of SDK and arcade in global.json"
arcadeName=$(tar -tf $finalSdkDir/packages/archive/Private.SourceBuilt.Artifacts*.tar.gz Microsoft.DotNet.Arcade.Sdk.*.nupkg)
arcadeVer=${arcadeName//Microsoft.DotNet.Arcade.Sdk.}
arcadeVer=${arcadeVer//.nupkg}
LogMessage " Arcade version = [$arcadeVer]"
sed -i "s|\(\"Microsoft.DotNet.Arcade.Sdk\": \"\)[^\"]*\"|\1$arcadeVer\"|g" $finalSdkDir/global.json

sdkVer=$(ls $finalSdkDir/.dotnet/sdk)
LogMessage " Sdk version = [$sdkVer]"
sed -i "s|\"dotnet\": \".*|\"dotnet\": \"$sdkVer\"|g" $finalSdkDir/global.json
LogMessage "Install previously source-built archive from $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts*.tar.gz to $stage1SourceBuiltArtifactsInstallDir"
mkdir -p $stage1SourceBuiltArtifactsInstallDir
tar -xzf $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts*.tar.gz --directory $stage1SourceBuiltArtifactsInstallDir

LogMessage "Build final sdk"
pushd $finalSdkDir
./build.sh
./build.sh --with-ref-packages $refPkgsDir/artifacts/reference-packages --with-packages $stage1SourceBuiltArtifactsInstallDir --with-sdk $stage1SdkInstallDir
popd

LogMessage "Completing Step3 - Build final sdk"
Expand Down Expand Up @@ -205,6 +166,8 @@ refPkgSourceDir="$(abspath $3)"
startStep="${4:-}"

stage1Dir="$bootstrapDir/stage1-sdk"
stage1SdkInstallDir="$bootstrapDir/stage1-sdk-install"
stage1SourceBuiltArtifactsInstallDir="$bootstrapDir/stage1-source-built-artifacts-install"
refPkgsDir="$bootstrapDir/reference-packages"
finalSdkDir="$bootstrapDir/final-sdk"

Expand Down