Add macOS/arm64 cross-compilation Azure CI job incl. prebuilt package

.azure-pipelines/posix-1-setup.yml

@@ -0,0 +1,91 @@
+# Clones the git repos, installs prerequisites and builds the first (bootstrap)
+# compiler.
+#
+# Required env vars:
+# - BOOTSTRAP_CMAKE_FLAGS (optional)
+# - CI_OS
+# - CLANG_VERSION
+# - HOST_LDC_VERSION
+# - LLVM_VERSION
+# Exported env var:
+# - PARALLEL_JOBS
+
+steps:
+
+# Clone
+- checkout: self
+  submodules: true
+  fetchDepth: 50
+
+# Install prerequisites
+- script: |
+    set -ex
+    cd ..
+    if [ "$CI_OS" = "osx" ]; then
+      # Download & extract Ninja
+      curl -L -o ninja-mac.zip https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-mac.zip
+      mkdir ninja
+      tar -xf ninja-mac.zip -C ninja
+    else
+      export DEBIAN_FRONTEND=noninteractive
+      sudo dpkg --add-architecture i386
+      sudo apt-get -q update
+      sudo apt-get -yq install \
+        git-core cmake ninja-build g++-multilib \
+        libcurl3 libcurl3:i386 \
+        curl gdb p7zip-full tzdata unzip zip
+      # Download & extract clang
+      curl -L -o clang.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-$CLANG_VERSION/clang+llvm-$CLANG_VERSION-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+      mkdir clang
+      tar -xf clang.tar.xz --strip 1 -C clang
+      # Use ld.gold per default, so that LTO is tested
+      sudo update-alternatives --install /usr/bin/ld ld /usr/bin/ld.gold 99
+    fi
+    # Install lit
+    curl -OL https://bootstrap.pypa.io/2.7/get-pip.py
+    python get-pip.py --user
+    python -m pip install --user lit
+    python -c "import lit.main; lit.main.main();" --version . | head -n 1
+    # Download & extract host LDC
+    HOST_OS="$CI_OS"
+    if [ "$CI_OS" = "android" ]; then HOST_OS="linux"; fi
+    curl -L -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-$HOST_OS-x86_64.tar.xz
+    mkdir host-ldc
+    tar -xf ldc2.tar.xz --strip 1 -C host-ldc
+    # Download & extract LDC-flavoured LLVM
+    assertsSuffix=""
+    if [ "${BUILD_SOURCEBRANCH:0:10}" != "refs/tags/" ]; then
+      assertsSuffix="-withAsserts"
+    fi
+    curl -L -o llvm.tar.xz https://github.com/ldc-developers/llvm-project/releases/download/ldc-v$LLVM_VERSION/llvm-$LLVM_VERSION-$HOST_OS-x86_64$assertsSuffix.tar.xz
+    mkdir llvm
+    tar -xf llvm.tar.xz --strip 1 -C llvm
+    # Set PARALLEL_JOBS env variable and persist it for future steps
+    if [ "$CI_OS" = "osx" ]; then
+      PARALLEL_JOBS=$(sysctl -n hw.logicalcpu)
+    else
+      PARALLEL_JOBS=$(nproc)
+    fi
+    set +x
+    echo "##vso[task.setvariable variable=PARALLEL_JOBS]$PARALLEL_JOBS"
+  displayName: Install prerequisites
+
+# Build bootstrap LDC
+- script: |
+    set -ex
+    cd ..
+    export PATH="$PWD/ninja:$PATH" # for macOS
+    cmake --version
+    ninja --version
+    mkdir bootstrap-ldc
+    cd bootstrap-ldc
+    cmake -G Ninja $BUILD_SOURCESDIRECTORY \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_ROOT_DIR=$PWD/../llvm \
+      -DD_COMPILER=$PWD/../host-ldc/bin/ldmd2 \
+      -DBUILD_SHARED_LIBS=OFF \
+      -DLDC_LINK_MANUALLY=OFF \
+      $BOOTSTRAP_CMAKE_FLAGS
+    ninja -j$PARALLEL_JOBS
+    bin/ldc2 -version
+  displayName: Build bootstrap LDC

.azure-pipelines/posix-2-build_cross_android.yml

@@ -0,0 +1,82 @@
+# Cross-compiles LDC and default libraries to an Android target.
+#
+# Required env vars:
+# - ANDROID_NDK_VERSION
+# - ARCH
+# - CI_OS
+# - EXTRA_CMAKE_FLAGS (optional)
+# - EXTRA_CMAKE_FLAGS_CROSS (optional)
+# - LLVM_VERSION
+# - PARALLEL_JOBS
+# Exported env vars:
+# - DFLAGS
+# - EXTRA_CMAKE_FLAGS_ANDROID
+
+steps:
+
+# Set up cross-compilation
+- script: |
+    set -ex
+    cd ..
+    # Download & extract LDC-flavoured LLVM for Android target
+    curl -L -o llvm-$ARCH.tar.xz https://github.com/ldc-developers/llvm-project/releases/download/ldc-v$LLVM_VERSION/llvm-$LLVM_VERSION-$CI_OS-$ARCH.tar.xz
+    mkdir llvm-$ARCH
+    tar -xf llvm-$ARCH.tar.xz --strip 1 -C llvm-$ARCH
+    # Download & extract Android NDK
+    curl -OL https://dl.google.com/android/repository/android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip
+    unzip android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip > /dev/null
+    rm android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip
+    # The NDK toolchain file enforces `-g` as base C[XX] flag - remove it to
+    # *significantly* reduce executable sizes
+    TOOLCHAIN_FILE=$PWD/android-ndk-$ANDROID_NDK_VERSION/build/cmake/android.toolchain.cmake
+    sed -i 's|^  -g$||' $TOOLCHAIN_FILE
+    # Set EXTRA_CMAKE_FLAGS_ANDROID for future steps
+    set +x
+    echo "##vso[task.setvariable variable=EXTRA_CMAKE_FLAGS_ANDROID]-DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN_FILE"
+    set -x
+    # Add a llvm-config bash script (cannot run ARM executable)
+    androidEnv="android"
+    if [ "$ARCH" = "armv7a" ]; then androidEnv="androideabi"; fi
+    LLVM_TRIPLE=$ARCH--linux-$androidEnv
+    sed $BUILD_SOURCESDIRECTORY/.azure-pipelines/android-llvm-config.in \
+      -e "s|@LLVM_VERSION@|$LLVM_VERSION|g" \
+      -e "s|@LLVM_INSTALL_DIR@|$PWD/llvm-$ARCH|g" \
+      -e "s|@LLVM_DEFAULT_TARGET_TRIPLE@|$LLVM_TRIPLE|g" \
+      -e "s|@LLVM_TARGETS@|AArch64 ARM X86 WebAssembly|g" > llvm-$ARCH/bin/llvm-config
+    chmod 755 llvm-$ARCH/bin/llvm-config
+    # Set up DFLAGS for cross-compiling/linking with host ldmd2
+    DFLAGS="-mtriple=$LLVM_TRIPLE -L-L$PWD/ldc-build-runtime.tmp/lib -gcc=$PWD/android-ndk-$ANDROID_NDK_VERSION/toolchains/llvm/prebuilt/linux-x86_64/bin/$ARCH-linux-${androidEnv}21-clang"
+    set +x
+    echo "##vso[task.setvariable variable=DFLAGS]$DFLAGS"
+  displayName: Set up cross-compilation
+
+# Cross-compile default libraries
+- script: |
+    set -ex
+    cd ..
+    IFS=$'\n' extraFlags=( $(xargs -n1 <<<"$EXTRA_CMAKE_FLAGS $EXTRA_CMAKE_FLAGS_CROSS $EXTRA_CMAKE_FLAGS_ANDROID") )
+    bootstrap-ldc/bin/ldc-build-runtime --ninja -j $PARALLEL_JOBS \
+      --dFlags="-fvisibility=hidden;${DFLAGS// /;}" \
+      --targetSystem='Android;Linux;UNIX' \
+      --ldcSrcDir=$BUILD_SOURCESDIRECTORY \
+      "${extraFlags[@]#-D}"
+  displayName: Cross-compile default libraries
+
+# Cross-compile LDC executables
+- script: |
+    set -ex
+    cd ..
+    installDir=$PWD/install
+    mkdir build-$ARCH
+    cd build-$ARCH
+    IFS=$'\n' extraFlags=( $(xargs -n1 <<<"$EXTRA_CMAKE_FLAGS $EXTRA_CMAKE_FLAGS_CROSS $EXTRA_CMAKE_FLAGS_ANDROID") )
+    cmake -G Ninja $BUILD_SOURCESDIRECTORY \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_ROOT_DIR=$PWD/../llvm-$ARCH \
+      -DD_COMPILER=$PWD/../bootstrap-ldc/bin/ldmd2 \
+      -DCMAKE_INSTALL_PREFIX=$installDir \
+      -DINCLUDE_INSTALL_DIR=$installDir/import \
+      -DD_LINKER_ARGS="-fuse-ld=bfd;-L$PWD/../ldc-build-runtime.tmp/lib;-lphobos2-ldc;-ldruntime-ldc" \
+      "${extraFlags[@]}"
+    ninja -j$PARALLEL_JOBS ldc2 ldmd2 ldc-build-runtime ldc-profdata ldc-prune-cache
+  displayName: 'Cross-compile LDC executables'

.azure-pipelines/posix-2-build_cross_mac.yml

@@ -0,0 +1,63 @@
+# Cross-compiles LDC and default libraries to the macOS/arm64 target.
+#
+# Required env vars:
+# - ARCH
+# - CI_OS
+# - EXTRA_CMAKE_FLAGS (optional)
+# - EXTRA_CMAKE_FLAGS_CROSS (optional)
+# - LLVM_VERSION
+# - PARALLEL_JOBS
+# Exported env vars:
+# - DFLAGS
+
+steps:
+
+# Set up cross-compilation
+- script: |
+    set -ex
+    cd ..
+    # Download & extract LDC-flavoured LLVM for macOS target
+    curl -L -o llvm-$ARCH.tar.xz https://github.com/ldc-developers/llvm-project/releases/download/ldc-v$LLVM_VERSION/llvm-$LLVM_VERSION-$CI_OS-$ARCH.tar.xz
+    mkdir llvm-$ARCH
+    tar -xf llvm-$ARCH.tar.xz --strip 1 -C llvm-$ARCH
+    # Replace llvm-config by the native LLVM's
+    cp llvm/bin/llvm-config llvm-$ARCH/bin
+    # Set up DFLAGS for cross-compiling/linking with host ldmd2
+    triple=$ARCH-apple-macos
+    DFLAGS="-mtriple=$triple -L-L$PWD/ldc-build-runtime.tmp/lib -Xcc=-target -Xcc=$triple -Xcc=-isysroot -Xcc=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"
+    set +x
+    echo "##vso[task.setvariable variable=DFLAGS]$DFLAGS"
+  displayName: Set up cross-compilation
+
+# Cross-compile default libraries
+- script: |
+    set -ex
+    cd ..
+    export PATH="$PWD/ninja:$PATH"
+    IFS=$'\n' extraFlags=( $(xargs -n1 <<<"$EXTRA_CMAKE_FLAGS $EXTRA_CMAKE_FLAGS_CROSS" | cut -b3-) )
+    bootstrap-ldc/bin/ldc-build-runtime --ninja -j $PARALLEL_JOBS \
+      --dFlags="${DFLAGS// /;}" \
+      --ldcSrcDir=$BUILD_SOURCESDIRECTORY \
+      "${extraFlags[@]}"
+  displayName: Cross-compile default libraries
+
+# Cross-compile LDC executables
+- script: |
+    set -ex
+    cd ..
+    export PATH="$PWD/ninja:$PATH"
+    installDir=$PWD/install
+    mkdir build-$ARCH
+    cd build-$ARCH
+    IFS=$'\n' extraFlags=( $(xargs -n1 <<<"$EXTRA_CMAKE_FLAGS $EXTRA_CMAKE_FLAGS_CROSS") )
+    cmake -G Ninja $BUILD_SOURCESDIRECTORY \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_ROOT_DIR=$PWD/../llvm-$ARCH \
+      -DD_COMPILER=$PWD/../bootstrap-ldc/bin/ldmd2 \
+      -DCMAKE_INSTALL_PREFIX=$installDir \
+      -DINCLUDE_INSTALL_DIR=$installDir/import \
+      -DLDC_LINK_MANUALLY=OFF \
+      -DLDC_ENABLE_ASSERTIONS=OFF \
+      "${extraFlags[@]}"
+    ninja -j$PARALLEL_JOBS ldc2 ldmd2 ldc-build-runtime ldc-profdata ldc-prune-cache
+  displayName: 'Cross-compile LDC executables'

.azure-pipelines/posix-2-build_native.yml

@@ -0,0 +1,34 @@
+# Performs a full native build incl. all test runners.
+#
+# Required env vars:
+# - CI_OS
+# - EXTRA_CMAKE_FLAGS (optional)
+# - PARALLEL_JOBS
+
+steps:
+
+# Build LDC & LDC D unittests & defaultlib unittest runners
+- script: |
+    set -ex
+    cd ..
+    export PATH="$PWD/ninja:$PATH" # for macOS
+    # Linux: use clang instead of gcc, for LTO
+    if [ "$CI_OS" = "linux" ]; then
+      export CC=$PWD/clang/bin/clang
+      export CXX=$PWD/clang/bin/clang++
+    fi
+    installDir=$PWD/install
+    mkdir build
+    cd build
+    IFS=$'\n' extraFlags=( $(xargs -n1 <<<"$EXTRA_CMAKE_FLAGS") )
+    cmake -G Ninja $BUILD_SOURCESDIRECTORY \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_ROOT_DIR=$PWD/../llvm \
+      -DD_COMPILER=$PWD/../bootstrap-ldc/bin/ldmd2 \
+      -DCMAKE_INSTALL_PREFIX=$installDir \
+      -DINCLUDE_INSTALL_DIR=$installDir/import \
+      -DLDC_LINK_MANUALLY=OFF \
+      "${extraFlags[@]}"
+    ninja -j$PARALLEL_JOBS all ldc2-unittest all-test-runners
+    bin/ldc2 -version
+  displayName: Build LDC & LDC D unittests & defaultlib unittest runners

.azure-pipelines/posix-3-test.yml

@@ -0,0 +1,23 @@
+# Runs the full test suite.
+#
+# Required env vars:
+# - PARALLEL_JOBS
+
+steps:
+
+- script: cd ../build && ctest --output-on-failure -R "ldc2-unittest"
+  displayName: Run LDC D unittests
+  condition: succeededOrFailed()
+
+# Temporarily add LLVM bin dir to PATH, so that e.g. wasm-ld is found
+- script: cd ../build && PATH=$PWD/../llvm/bin:$PATH ctest -V -R "lit-tests"
+  displayName: Run LIT testsuite
+  condition: succeededOrFailed()
+
+- script: cd ../build && DMD_TESTSUITE_MAKE_ARGS=-j$PARALLEL_JOBS ctest -V -R "dmd-testsuite"
+  displayName: Run DMD testsuite
+  condition: succeededOrFailed()
+
+- script: cd ../build && ctest -j$PARALLEL_JOBS --output-on-failure -E "dmd-testsuite|lit-tests|ldc2-unittest"
+  displayName: Run defaultlib unittests & druntime integration tests
+  condition: succeededOrFailed()

.azure-pipelines/posix-4-install.yml

@@ -0,0 +1,37 @@
+# Installs LDC and makes the installation portable.
+#
+# Required env vars:
+# - ARCH
+# - EXTRA_CMAKE_FLAGS_CROSS (optional)
+
+steps:
+- script: |
+    set -ex
+    cd ..
+    if [[ "$EXTRA_CMAKE_FLAGS_CROSS" != "" ]]; then
+      mkdir -p install/bin
+      cp build-$ARCH/bin/{ldc2,ldmd2,ldc-build-runtime,ldc-profdata,ldc-prune-cache} install/bin
+      cp -R ldc-build-runtime.tmp/lib install
+      cp build-$ARCH/lib/{libldc_rt.*,libLTO-ldc.dylib,LLVMgold-ldc.so} install/lib || true
+      mkdir install/etc
+      cp build-$ARCH/bin/ldc2_install.conf install/etc/ldc2.conf
+      cp -R $BUILD_SOURCESDIRECTORY/packaging/bash_completion.d install/etc
+      mkdir install/import
+      cp -R $BUILD_SOURCESDIRECTORY/runtime/druntime/src/{core,etc,ldc,object.d} install/import
+      cp bootstrap-ldc/runtime/import/ldc/gccbuiltins_*.di install/import/ldc
+      cp -R $BUILD_SOURCESDIRECTORY/runtime/phobos/etc/c install/import/etc
+      rm -rf install/import/etc/c/zlib
+      cp -R $BUILD_SOURCESDIRECTORY/runtime/phobos/std install/import
+    else
+      export PATH="$PWD/ninja:$PATH"
+      cd build
+      ninja install > /dev/null
+      cd ..
+    fi
+    cp $BUILD_SOURCESDIRECTORY/{LICENSE,packaging/README} install
+    # Make portable
+    perl -pi -e "s|$PWD/install/|%%ldcbinarypath%%/../|g" install/etc/ldc2.conf
+    cat install/etc/ldc2.conf
+    # Now rename the installation dir to test portability
+    mv install installed
+  displayName: Install LDC & make portable

.azure-pipelines/posix-4a-add_android_x86.yml

@@ -0,0 +1,28 @@
+# Cross-compiles the Android x86 default libraries and copies them to the
+# install dir.
+#
+# Required env vars:
+# - ANDROID_X86_ABI
+# - EXTRA_CMAKE_FLAGS (optional)
+# - EXTRA_CMAKE_FLAGS_CROSS (optional)
+# - EXTRA_CMAKE_FLAGS_ANDROID (optional)
+# - PARALLEL_JOBS
+
+steps:
+- script: |
+    set -ex
+    cd ..
+    arch=$ANDROID_X86_ABI
+    if [ "$arch" = "x86" ]; then arch="i686"; fi
+    bootstrap-ldc/bin/ldc-build-runtime --ninja -j $PARALLEL_JOBS \
+      --buildDir=build-libs-$arch \
+      --dFlags="-fvisibility=hidden;-mtriple=$arch-linux-android" \
+      --targetSystem='Android;Linux;UNIX' \
+      --ldcSrcDir=$BUILD_SOURCESDIRECTORY \
+      ${EXTRA_CMAKE_FLAGS//-D/} \
+      ${EXTRA_CMAKE_FLAGS_CROSS//-D/} \
+      ${EXTRA_CMAKE_FLAGS_ANDROID//-D/} \
+      ANDROID_ABI=$ANDROID_X86_ABI
+    mkdir installed/lib-$arch
+    cp build-libs-$arch/lib/*.a installed/lib-$arch
+  displayName: 'Cross-compile x86 default libraries and copy to install dir'