Skip to content

Commit 88fd32c

Browse files
MaxDesiatovMaxDesiatov
authored
Merge pull request #1024 from swiftwasm/maxd/build-host-toolchain
Build host toolchain w/o copying it from snapshots Currently the host toolchain is copied over to the target WASI toolchain from snapshots downloaded from [swift.org](https://swift.org/download/#snapshots). This is not ideal as snapshots on swift.org are not always fresh and don't always match what we have in the `swiftwasm` branch or even in the upstream `master` branch. This causes issues similar to what can be seen in #1000, where the host toolchain and SDK don't match the target toolchain with this error: ``` error: compiled module was created by an older version of the compiler; rebuild 'Swift' and try again: /usr/lib/swift/macosx/Swift.swiftmodule/x86_64-apple-macos.swiftmodule ``` The solution is to: 1) build the host toolchain together with the host SDK and SwiftPM as the first part of the CI process 2) install the host products into the `$SOURCE_PATH/host-toolchain-sdk` directory 3) clean up the build directory 4) build the target SwiftWasm SDK 5) install the target SwiftWasm SDK into the `$SOURCE_PATH/install` directory 6) copy over WASI SDK clang/llvm-ar/wasm-ld into the `$SOURCE_PATH/host-toolchain-sdk` directory 6) copy over the target SwiftWasm SDK into the `$SOURCE_PATH/host-toolchain-sdk` directory 7) create an archive from the `$SOURCE_PATH/host-toolchain-sdk` directory Thus `build-swiftpm.sh` script is no longer needed as SwiftPM is already compiled as a part of the host toolchain, which also avoids similar issues where SwiftPM from upstream snapshots can also be out of sync from what we build here on CI.
2 parents 04c8aba + 080a388 commit 88fd32c

File tree

4 files changed

+88
-173
lines changed

4 files changed

+88
-173
lines changed

‎utils/build-presets.ini

+50-34
Original file line numberDiff line numberDiff line change
@@ -2437,39 +2437,67 @@ no-assertions
24372437

24382438
[preset: webassembly]
24392439

2440-
wasm
24412440
release
2442-
verbose
24432441
cmake-c-launcher=%(C_CXX_LAUNCHER)s
24442442
cmake-cxx-launcher=%(C_CXX_LAUNCHER)s
24452443
skip-build-benchmarks
2444+
llvm-targets-to-build=X86;WebAssembly
2445+
install-destdir=%(INSTALL_DESTDIR)s
2446+
swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
2447+
llvm-install-components=clang
2448+
install-swift
2449+
install-prefix=/%(TOOLCHAIN_NAME)s/usr
2450+
2451+
[preset: webassembly-host]
2452+
2453+
mixin-preset=webassembly
2454+
extra-cmake-options=
2455+
-DSWIFT_BUILD_SOURCEKIT=FALSE
2456+
-DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
2457+
-DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
2458+
2459+
llbuild
2460+
swiftpm
2461+
2462+
install-swift
2463+
install-llbuild
2464+
install-swiftpm
2465+
2466+
[preset: webassembly-linux-host]
2467+
2468+
mixin-preset=webassembly-host
2469+
2470+
libdispatch
2471+
libicu
2472+
foundation
2473+
xctest
2474+
2475+
install-libicu
2476+
install-foundation
2477+
install-libdispatch
2478+
install-xctest
2479+
2480+
skip-test-libicu
2481+
skip-test-foundation
2482+
skip-test-libdispatch
2483+
skip-test-xctest
2484+
2485+
[preset: webassembly-target]
2486+
2487+
mixin-preset=webassembly
2488+
wasm
2489+
verbose
24462490
build-stdlib-deployment-targets=wasi-wasm32
24472491
build-swift-dynamic-sdk-overlay=false
24482492
build-swift-dynamic-stdlib=false
24492493
build-swift-static-sdk-overlay
24502494
build-swift-static-stdlib
2451-
llvm-targets-to-build=X86;WebAssembly
24522495
stdlib-deployment-targets=wasi-wasm32
24532496
wasi-sdk=%(SOURCE_PATH)s/wasi-sdk
24542497

2455-
[preset: webassembly-installable]
2498+
[preset: webassembly-linux-target]
24562499

2457-
install_destdir=%(SOURCE_PATH)s/install
2458-
installable_package=%(INSTALLABLE_PACKAGE)s
2459-
install-prefix=/%(TOOLCHAIN_NAME)s/usr
2460-
swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
2461-
llvm-install-components=clang
2462-
install-swift
2463-
darwin-toolchain-bundle-identifier=%(BUNDLE_IDENTIFIER)s
2464-
darwin-toolchain-display-name=%(DISPLAY_NAME)s
2465-
darwin-toolchain-display-name-short=%(DISPLAY_NAME_SHORT)s
2466-
darwin-toolchain-name=%(TOOLCHAIN_NAME)s
2467-
darwin-toolchain-version=%(TOOLCHAIN_VERSION)s
2468-
darwin-toolchain-alias=swift
2469-
2470-
[preset: webassembly-linux]
2471-
2472-
mixin-preset=webassembly
2500+
mixin-preset=webassembly-target
24732501
extra-cmake-options=
24742502
-DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
24752503
-DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
@@ -2482,9 +2510,9 @@ extra-cmake-options=
24822510
-DCMAKE_AR="%(SOURCE_PATH)s/wasi-sdk/bin/llvm-ar"
24832511
-DCMAKE_RANLIB="%(SOURCE_PATH)s/wasi-sdk/bin/llvm-ranlib"
24842512

2485-
[preset: webassembly-macos]
2513+
[preset: webassembly-macos-target]
24862514

2487-
mixin-preset=webassembly
2515+
mixin-preset=webassembly-target
24882516
extra-cmake-options=
24892517
-DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
24902518
-DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
@@ -2496,15 +2524,3 @@ extra-cmake-options=
24962524
-DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
24972525
-DCMAKE_AR='/usr/local/opt/llvm/bin/llvm-ar'
24982526
-DCMAKE_RANLIB='/usr/local/opt/llvm/bin/llvm-ranlib'
2499-
2500-
[preset: webassembly-linux-installable]
2501-
2502-
mixin-preset=
2503-
webassembly-linux
2504-
webassembly-installable
2505-
2506-
[preset: webassembly-macos-installable]
2507-
2508-
mixin-preset=
2509-
webassembly-macos
2510-
webassembly-installable

‎utils/webassembly/build-swiftpm.sh

-60
This file was deleted.

‎utils/webassembly/build-toolchain.sh

+38-29
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@ WASI_SDK_PATH=$SOURCE_PATH/wasi-sdk
99
case $(uname -s) in
1010
Darwin)
1111
OS_SUFFIX=osx
12-
PRESET_NAME=webassembly-macos-installable
12+
HOST_PRESET=webassembly-host
13+
TARGET_PRESET=webassembly-macos-target
1314
;;
1415
Linux)
1516
OS_SUFFIX=linux
16-
PRESET_NAME=webassembly-linux-installable
17+
HOST_PRESET=webassembly-linux-host
18+
TARGET_PRESET=webassembly-linux-target
1719
;;
1820
*)
1921
echo "Unrecognised platform $(uname -s)"
@@ -35,47 +37,54 @@ BUNDLE_IDENTIFIER="swiftwasm.${YEAR}${MONTH}${DAY}"
3537
DISPLAY_NAME_SHORT="Swift for WebAssembly Development Snapshot"
3638
DISPLAY_NAME="${DISPLAY_NAME_SHORT} ${YEAR}-${MONTH}-${DAY}"
3739

38-
$SOURCE_PATH/swift/utils/build-script --preset=$PRESET_NAME \
40+
HOST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/host-toolchain-sdk
41+
HOST_TOOLCHAIN_SDK=$HOST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
42+
43+
# Avoid clang headers symlink issues
44+
mkdir -p $HOST_TOOLCHAIN_SDK/usr/lib/clang/10.0.0
45+
46+
# Build the host toolchain and SDK first.
47+
$SOURCE_PATH/swift/utils/build-script \
48+
--preset=$HOST_PRESET \
49+
INSTALL_DESTDIR="$HOST_TOOLCHAIN_DESTDIR" \
50+
TOOLCHAIN_NAME="$TOOLCHAIN_NAME" \
51+
C_CXX_LAUNCHER="$(which sccache)"
52+
53+
# Clean up the host toolchain build directory so that the next
54+
# `build-script` invocation doesn't pick up wrong CMake config files.
55+
# For some reason passing `--reconfigure` to `build-script` won't do this.
56+
rm -rf $SOURCE_PATH/build/Ninja-ReleaseAssert/swift-*
57+
58+
# build the cross-compilled toolchain
59+
$SOURCE_PATH/swift/utils/build-script \
60+
--preset=$TARGET_PRESET \
61+
INSTALL_DESTDIR="$SOURCE_PATH/install" \
3962
SOURCE_PATH="$SOURCE_PATH" \
40-
INSTALLABLE_PACKAGE="$INSTALLABLE_PACKAGE" \
4163
BUNDLE_IDENTIFIER="${BUNDLE_IDENTIFIER}" \
4264
DISPLAY_NAME="${DISPLAY_NAME}" \
4365
DISPLAY_NAME_SHORT="${DISPLAY_NAME_SHORT}" \
4466
TOOLCHAIN_NAME="${TOOLCHAIN_NAME}" \
4567
TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \
4668
C_CXX_LAUNCHER="$(which sccache)"
4769

48-
49-
NIGHTLY_TOOLCHAIN=$SOURCE_PATH/swift-nightly-toolchain
50-
if [ ! -e $NIGHTLY_TOOLCHAIN ]; then
51-
$UTILS_PATH/install-nightly-toolchain.sh
52-
fi
53-
54-
TMP_DIR=$(mktemp -d)
55-
cd $TMP_DIR
56-
tar xfz $INSTALLABLE_PACKAGE $TOOLCHAIN_NAME
57-
cd $TMP_DIR/$TOOLCHAIN_NAME
58-
59-
# Merge wasi-sdk and toolchain
60-
cp -r $WASI_SDK_PATH/lib/clang usr/lib
61-
cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} usr/bin
62-
cp -r $WASI_SDK_PATH/share/wasi-sysroot usr/share
63-
64-
# Build SwiftPM and install it into toolchain
65-
$UTILS_PATH/build-swiftpm.sh $TMP_DIR/$TOOLCHAIN_NAME
70+
# Merge wasi-sdk and the toolchain
71+
cp -a $WASI_SDK_PATH/lib/clang $HOST_TOOLCHAIN_SDK/usr/lib
72+
cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} $HOST_TOOLCHAIN_SDK/usr/bin
73+
cp -r $WASI_SDK_PATH/share/wasi-sysroot $HOST_TOOLCHAIN_SDK/usr/share
6674

6775
# Replace absolute sysroot path with relative path
68-
sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
76+
sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
6977

70-
# Copy nightly-toolchain's host environment stdlib into toolchain
78+
# Copy the target environment stdlib into the toolchain
7179

7280
if [[ "$(uname)" == "Linux" ]]; then
73-
# Avoid to copy usr/lib/swift/clang because our toolchain's one is a directory
74-
# but nightly's one is symbolic link, so fail to merge them.
75-
rsync -a $NIGHTLY_TOOLCHAIN/usr/lib/ $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/ --exclude 'swift/clang'
81+
# Avoid copying usr/lib/swift/clang because our toolchain's one is a directory
82+
# but nightly's one is symbolic link. A simple copy fails to merge them.
83+
rsync -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/ $HOST_TOOLCHAIN_SDK/usr/lib/ --exclude 'swift/clang'
7684
else
77-
cp -r $NIGHTLY_TOOLCHAIN/usr/lib/swift/macosx $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/swift
85+
cp -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift_static $HOST_TOOLCHAIN_SDK/usr/lib/swift_static
86+
cp -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift/wasi $HOST_TOOLCHAIN_SDK/usr/lib/swift
7887
fi
7988

80-
cd $TMP_DIR
89+
cd $HOST_TOOLCHAIN_DESTDIR
8190
tar cfz $PACKAGE_ARTIFACT $TOOLCHAIN_NAME

‎utils/webassembly/install-nightly-toolchain.sh

-50
This file was deleted.

0 commit comments

Comments
 (0)