From 8368a35f834dc56519683ba3d98576ec3e905eb4 Mon Sep 17 00:00:00 2001 From: "Joshua M. Clulow" Date: Wed, 8 Jul 2020 16:39:09 -0700 Subject: [PATCH] build dist for x86_64-unknown-illumos This change creates a new Docker image, "dist-x86_64-illumos", and sets things up to build the full set of "dist" packages for illumos hosts, so that illumos users can use "rustup" to install packages. It also adjusts the manifest builder to expect complete toolchains for this platform. --- .github/workflows/ci.yml | 3 + src/ci/azure-pipelines/auto.yml | 1 + .../dist-x86_64-illumos/Dockerfile | 33 ++++ src/ci/docker/scripts/illumos-toolchain.sh | 177 ++++++++++++++++++ src/ci/github-actions/ci.yml | 3 + src/tools/build-manifest/src/main.rs | 1 + 6 files changed, 218 insertions(+) create mode 100644 src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile create mode 100644 src/ci/docker/scripts/illumos-toolchain.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ffc8a7ad5241..66f201db01acb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -309,6 +309,9 @@ jobs: - name: dist-x86_64-freebsd os: ubuntu-latest-xl env: {} + - name: dist-x86_64-illumos + os: ubuntu-latest-xl + env: {} - name: dist-x86_64-linux os: ubuntu-latest-xl env: {} diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 3de5a19f85164..0b460bbe865ee 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -55,6 +55,7 @@ jobs: dist-powerpc64le-linux: {} dist-s390x-linux: {} dist-x86_64-freebsd: {} + dist-x86_64-illumos: {} dist-x86_64-musl: {} dist-x86_64-netbsd: {} i686-gnu: {} diff --git a/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile new file mode 100644 index 0000000000000..2a0f6a39c5781 --- /dev/null +++ b/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:18.04 + +# Enable source repositories, which are disabled by default on Ubuntu >= 18.04 +RUN sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list + +COPY scripts/cross-apt-packages.sh /tmp/ +RUN bash /tmp/cross-apt-packages.sh + +# Required for cross-build gcc +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + libgmp-dev \ + libmpfr-dev \ + libmpc-dev + +COPY scripts/illumos-toolchain.sh /tmp/ + +RUN bash /tmp/illumos-toolchain.sh x86_64 sysroot +RUN bash /tmp/illumos-toolchain.sh x86_64 binutils +RUN bash /tmp/illumos-toolchain.sh x86_64 gcc + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV \ + AR_x86_64_unknown_illumos=x86_64-illumos-ar \ + CC_x86_64_unknown_illumos=x86_64-illumos-gcc \ + CXX_x86_64_unknown_illumos=x86_64-illumos-g++ + +ENV HOSTS=x86_64-unknown-illumos + +ENV RUST_CONFIGURE_ARGS --enable-extended --disable-docs +ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS diff --git a/src/ci/docker/scripts/illumos-toolchain.sh b/src/ci/docker/scripts/illumos-toolchain.sh new file mode 100644 index 0000000000000..8cb5712657947 --- /dev/null +++ b/src/ci/docker/scripts/illumos-toolchain.sh @@ -0,0 +1,177 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o xtrace + +ARCH="$1" +PHASE="$2" + +JOBS="$(getconf _NPROCESSORS_ONLN)" + +case "$ARCH" in +x86_64) + SYSROOT_MACH='i386' + ;; +*) + printf 'ERROR: unknown architecture: %s\n' "$ARCH" + exit 1 +esac + +BUILD_TARGET="$ARCH-sun-solaris2.10" + +# +# The illumos and the Solaris build both use the same GCC-level host triple, +# though different versions of GCC are used and with different configure +# options. To ensure as little accidental cross-pollination as possible, we +# build the illumos toolchain in a specific directory tree and just symlink the +# expected tools into /usr/local/bin at the end. We omit /usr/local/bin from +# PATH here for similar reasons. +# +PREFIX="/opt/illumos/$ARCH" +export PATH="$PREFIX/bin:/usr/bin:/bin:/usr/sbin:/sbin" + +# +# NOTE: The compiler version selected here is more specific than might appear. +# GCC 7.X releases do not appear to cross-compile correctly for Solaris +# targets, at least insofar as they refuse to enable TLS in libstdc++. When +# changing the GCC version in future, one must carefully verify that TLS is +# enabled in all of the static libraries we intend to include in output +# binaries. +# +GCC_VERSION='8.4.0' +GCC_SUM='e30a6e52d10e1f27ed55104ad233c30bd1e99cfb5ff98ab022dc941edd1b2dd4' +GCC_BASE="gcc-$GCC_VERSION" +GCC_TAR="gcc-$GCC_VERSION.tar.xz" +GCC_URL="https://ftp.gnu.org/gnu/gcc/$GCC_BASE/$GCC_TAR" + +SYSROOT_VER='20181213-de6af22ae73b-v1' +SYSROOT_SUM='ee792d956dfa6967453cebe9286a149143290d296a8ce4b8a91d36bea89f8112' +SYSROOT_TAR="illumos-sysroot-$SYSROOT_MACH-$SYSROOT_VER.tar.gz" +SYSROOT_URL='https://github.com/illumos/sysroot/releases/download/' +SYSROOT_URL+="$SYSROOT_VER/$SYSROOT_TAR" +SYSROOT_DIR="$PREFIX/sysroot" + +BINUTILS_VERSION='2.25.1' +BINUTILS_SUM='b5b14added7d78a8d1ca70b5cb75fef57ce2197264f4f5835326b0df22ac9f22' +BINUTILS_BASE="binutils-$BINUTILS_VERSION" +BINUTILS_TAR="$BINUTILS_BASE.tar.bz2" +BINUTILS_URL="https://ftp.gnu.org/gnu/binutils/$BINUTILS_TAR" + + +download_file() { + local file="$1" + local url="$2" + local sum="$3" + + while :; do + if [[ -f "$file" ]]; then + if ! h="$(sha256sum "$file" | awk '{ print $1 }')"; then + printf 'ERROR: reading hash\n' >&2 + exit 1 + fi + + if [[ "$h" == "$sum" ]]; then + return 0 + fi + + printf 'WARNING: hash mismatch: %s != expected %s\n' \ + "$h" "$sum" >&2 + rm -f "$file" + fi + + printf 'Downloading: %s\n' "$url" + if ! curl -f -L -o "$file" "$url"; then + rm -f "$file" + sleep 1 + fi + done +} + + +case "$PHASE" in +sysroot) + download_file "/tmp/$SYSROOT_TAR" "$SYSROOT_URL" "$SYSROOT_SUM" + mkdir -p "$SYSROOT_DIR" + cd "$SYSROOT_DIR" + tar -xzf "/tmp/$SYSROOT_TAR" + rm -f "/tmp/$SYSROOT_TAR" + ;; + +binutils) + download_file "/tmp/$BINUTILS_TAR" "$BINUTILS_URL" "$BINUTILS_SUM" + mkdir -p /ws/src/binutils + cd /ws/src/binutils + tar -xjf "/tmp/$BINUTILS_TAR" + rm -f "/tmp/$BINUTILS_TAR" + + mkdir -p /ws/build/binutils + cd /ws/build/binutils + "/ws/src/binutils/$BINUTILS_BASE/configure" \ + --prefix="$PREFIX" \ + --target="$BUILD_TARGET" \ + --program-prefix="$ARCH-illumos-" \ + --with-sysroot="$SYSROOT_DIR" + + make -j "$JOBS" + + mkdir -p "$PREFIX" + make install + + cd / + rm -rf /ws/src/binutils /ws/build/binutils + ;; + +gcc) + download_file "/tmp/$GCC_TAR" "$GCC_URL" "$GCC_SUM" + mkdir -p /ws/src/gcc + cd /ws/src/gcc + tar -xJf "/tmp/$GCC_TAR" + rm -f "/tmp/$GCC_TAR" + + mkdir -p /ws/build/gcc + cd /ws/build/gcc + export CFLAGS='-fPIC' + export CXXFLAGS='-fPIC' + export CXXFLAGS_FOR_TARGET='-fPIC' + export CFLAGS_FOR_TARGET='-fPIC' + "/ws/src/gcc/$GCC_BASE/configure" \ + --prefix="$PREFIX" \ + --target="$BUILD_TARGET" \ + --program-prefix="$ARCH-illumos-" \ + --with-sysroot="$SYSROOT_DIR" \ + --with-gnu-as \ + --with-gnu-ld \ + --disable-nls \ + --disable-libgomp \ + --disable-libquadmath \ + --disable-libssp \ + --disable-libvtv \ + --disable-libcilkrts \ + --disable-libada \ + --disable-libsanitizer \ + --disable-libquadmath-support \ + --disable-shared \ + --enable-tls + + make -j "$JOBS" + + mkdir -p "$PREFIX" + make install + + # + # Link toolchain commands into /usr/local/bin so that cmake and others + # can find them: + # + (cd "$PREFIX/bin" && ls -U) | grep "^$ARCH-illumos-" | + xargs -t -I% ln -s "$PREFIX/bin/%" '/usr/local/bin/' + + cd / + rm -rf /ws/src/gcc /ws/build/gcc + ;; + +*) + printf 'ERROR: unknown phase "%s"\n' "$PHASE" >&2 + exit 100 + ;; +esac diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 0614d5e31d1ee..052ab37ecf3c8 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -353,6 +353,9 @@ jobs: - name: dist-x86_64-freebsd <<: *job-linux-xl + - name: dist-x86_64-illumos + <<: *job-linux-xl + - name: dist-x86_64-linux <<: *job-linux-xl diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 5c50587ea973d..2c64abb4be6bd 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -39,6 +39,7 @@ static HOSTS: &[&str] = &[ "x86_64-pc-windows-gnu", "x86_64-pc-windows-msvc", "x86_64-unknown-freebsd", + "x86_64-unknown-illumos", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", "x86_64-unknown-netbsd",