From c9025bd59575f598c1ac704fe7b497d849f26a26 Mon Sep 17 00:00:00 2001 From: Julian Ospald Date: Thu, 16 Sep 2021 12:47:05 +0200 Subject: [PATCH 1/4] Add gitlab CI --- .gitlab-ci.yml | 112 +++++++++++++++++++++++++++++++ .gitlab/ci.sh | 51 ++++++++++++++ .gitlab/common.sh | 49 ++++++++++++++ .gitlab/shell-aarch64-darwin.nix | 90 +++++++++++++++++++++++++ cabal.project | 2 + 5 files changed, 304 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100755 .gitlab/ci.sh create mode 100644 .gitlab/common.sh create mode 100644 .gitlab/shell-aarch64-darwin.nix diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..21773896a9 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,112 @@ +variables: + # Commit of ghc/ci-images repository from which to pull Docker images + DOCKER_REV: "853f348f9caf38b08740b280296fbd34e09abb3a" + + CABAL_INSTALL_VERSION: 3.4.0.0 + +.default_matrix: &default_matrix + matrix: + - GHC_VERSION: 8.8.4 + CABAL_PROJECT: cabal.project + - GHC_VERSION: 8.10.7 + CABAL_PROJECT: cabal.project + - GHC_VERSION: 9.0.1 + CABAL_PROJECT: cabal-ghc901.project + +.m1_matrix: &m1_matrix + matrix: + - GHC_VERSION: 8.10.7 + CABAL_PROJECT: cabal.project + +.arm_matrix: &arm_matrix + matrix: + - GHC_VERSION: 8.10.7 + CABAL_PROJECT: cabal.project + + +workflow: + rules: + - if: $CI_COMMIT_TAG + when: always + - when: never + +.build: + script: + - bash .gitlab/ci.sh + artifacts: + expire_in: 2 week + paths: + - out + +build-aarch64-linux-deb10: + extends: .build + tags: + - aarch64-linux + image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb10:$DOCKER_REV" + parallel: *arm_matrix + +build-armv7-linux-deb10: + extends: .build + tags: + - armv7-linux + image: "registry.gitlab.haskell.org/ghc/ci-images/armv7-linux-deb10:$DOCKER_REV" + parallel: *arm_matrix + +build-x86_64-linux: + extends: .build + tags: + - x86_64-linux + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV" + parallel: *default_matrix + +build-x86_64-linux-alpine: + extends: .build + tags: + - x86_64-linux + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-alpine3_12:$DOCKER_REV" + parallel: *default_matrix + +build-x86_64-freebsd: + extends: .build + tags: + - x86_64-freebsd13 + parallel: *default_matrix + +build-x86_64-darwin: + extends: .build + tags: + - x86_64-darwin + parallel: *default_matrix + +build-aarch64-darwin: + tags: + - aarch64-darwin-m1 + script: | + set -Eeuo pipefail + function runInNixShell() { + time nix-shell $CI_PROJECT_DIR/.gitlab/shell-aarch64-darwin.nix \ + -I nixpkgs=https://github.com/angerman/nixpkgs/archive/75f7281738b.tar.gz \ + --argstr system "aarch64-darwin" \ + --pure \ + --keep CI_PROJECT_DIR \ + --keep MACOSX_DEPLOYMENT_TARGET \ + --keep GHC_VERSION \ + --keep CABAL_PROJECT \ + --keep CABAL_INSTALL_VERSION \ + --run "$1" 2>&1 + } + runInNixShell "cabal update && mkdir vendored && cd vendored && cabal unpack network-3.1.2.1 && cd network-3.1.2.1 && autoreconf -fi" 2>&1 + runInNixShell "./.gitlab/ci.sh" 2>&1 + variables: + MACOSX_DEPLOYMENT_TARGET: "10.7" + parallel: *m1_matrix + artifacts: + expire_in: 2 week + paths: + - out + +build-x86_64-windows: + extends: .build + tags: + - new-x86_64-windows + parallel: *default_matrix diff --git a/.gitlab/ci.sh b/.gitlab/ci.sh new file mode 100755 index 0000000000..30133f8422 --- /dev/null +++ b/.gitlab/ci.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -Eeuxo pipefail + +source "$CI_PROJECT_DIR/.gitlab/common.sh" + + +export GHCUP_INSTALL_BASE_PREFIX="$CI_PROJECT_DIR/toolchain" +export CABAL_DIR="$CI_PROJECT_DIR/cabal" + +case "$(uname)" in + MSYS_*|MINGW*) + export CABAL_DIR="$(cygpath -w "$CABAL_DIR")" + GHCUP_BINDIR="${GHCUP_INSTALL_BASE_PREFIX}/ghcup/bin" + ;; + *) + GHCUP_BINDIR="${GHCUP_INSTALL_BASE_PREFIX}/.ghcup/bin" + ;; +esac + +mkdir -p "$CABAL_DIR" +mkdir -p "$GHCUP_BINDIR" +export PATH="$GHCUP_BINDIR:$PATH" + +export BOOTSTRAP_HASKELL_NONINTERACTIVE=1 +export BOOTSTRAP_HASKELL_GHC_VERSION="$GHC_VERSION" +export BOOTSTRAP_HASKELL_CABAL_VERSION="$CABAL_INSTALL_VERSION" +export BOOTSTRAP_HASKELL_VERBOSE=1 +export BOOTSTRAP_HASKELL_ADJUST_CABAL_CONFIG=yes + +curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh + +# some alpines need workaround +if ghc --info | grep -q integer-simple ; then + echo -e 'package blaze-textual\n flags: +integer-simple' >> cabal.project.local +fi + +run cabal v2-install exe:haskell-language-server exe:haskell-language-server-wrapper \ + -O2 \ + -w "ghc-$GHC_VERSION" \ + --project-file "$CABAL_PROJECT" \ + --installdir="$CI_PROJECT_DIR/out" \ + --install-method=copy \ + --overwrite-policy=always \ + --enable-executable-static \ + --disable-profiling \ + --disable-tests \ + --enable-split-sections \ + --enable-executable-stripping + +cp dist-newstyle/cache/plan.json "$CI_PROJECT_DIR/out/plan.json" diff --git a/.gitlab/common.sh b/.gitlab/common.sh new file mode 100644 index 0000000000..b6bce698c9 --- /dev/null +++ b/.gitlab/common.sh @@ -0,0 +1,49 @@ +# Common bash utilities +# ---------------------- + +# Colors +BLACK="0;30" +GRAY="1;30" +RED="0;31" +LT_RED="1;31" +BROWN="0;33" +LT_BROWN="1;33" +GREEN="0;32" +LT_GREEN="1;32" +BLUE="0;34" +LT_BLUE="1;34" +PURPLE="0;35" +LT_PURPLE="1;35" +CYAN="0;36" +LT_CYAN="1;36" +WHITE="1;37" +LT_GRAY="0;37" + +# GitLab Pipelines log section delimiters +# https://gitlab.com/gitlab-org/gitlab-foss/issues/14664 +start_section() { + name="$1" + echo -e "section_start:$(date +%s):$name\015\033[0K" +} + +end_section() { + name="$1" + echo -e "section_end:$(date +%s):$name\015\033[0K" +} + +echo_color() { + local color="$1" + local msg="$2" + echo -e "\033[${color}m${msg}\033[0m" +} + +error() { echo_color "${RED}" "$1"; } +warn() { echo_color "${LT_BROWN}" "$1"; } +info() { echo_color "${LT_BLUE}" "$1"; } + +fail() { error "error: $1"; exit 1; } + +function run() { + info "Running $*..." + "$@" || ( error "$* failed"; return 1; ) +} diff --git a/.gitlab/shell-aarch64-darwin.nix b/.gitlab/shell-aarch64-darwin.nix new file mode 100644 index 0000000000..33c7c67bee --- /dev/null +++ b/.gitlab/shell-aarch64-darwin.nix @@ -0,0 +1,90 @@ +{ system ? "aarch64-darwin" +#, nixpkgs ? fetchTarball https://github.com/angerman/nixpkgs/archive/257cb120334.tar.gz #apple-silicon.tar.gz +, pkgs ? import { inherit system; } +, compiler ? if system == "aarch64-darwin" then "ghc8103Binary" else "ghc8103" +}: pkgs.mkShell { + # this prevents nix from trying to write the env-vars file. + # we can't really, as NIX_BUILD_TOP/env-vars is not set. + noDumpEnvVars=1; + + # stop polluting LDFLAGS with -liconv + dontAddExtraLibs = true; + + # we need to inject ncurses into --with-curses-libraries. + # the real fix is to teach terminfo to use libcurses on macOS. + # CONFIGURE_ARGS = "--with-intree-gmp --with-curses-libraries=${pkgs.ncurses.out}/lib"; + CONFIGURE_ARGS = "--with-intree-gmp --with-curses-libraries=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib --with-iconv-includes=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include --with-iconv-libraries=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib SH=/bin/bash"; + + # magic speedup pony :facepalm: + # + # nix has the ugly habbit of duplicating ld flags more than necessary. This + # somewhat consolidates this. + shellHook = '' + export NIX_LDFLAGS=$(for a in $NIX_LDFLAGS; do echo $a; done |sort|uniq|xargs) + export NIX_LDFLAGS_FOR_TARGET=$(for a in $NIX_LDFLAGS_FOR_TARGET; do echo $a; done |sort|uniq|xargs) + export NIX_LDFLAGS_FOR_TARGET=$(comm -3 <(for l in $NIX_LDFLAGS_FOR_TARGET; do echo $l; done) <(for l in $NIX_LDFLAGS; do echo $l; done)) + + + # Impurity hack for GHC releases. + ################################# + # We don't want binary releases to depend on nix, thus we'll need to make sure we don't leak in references. + # GHC externally depends only on iconv and curses. However we can't force a specific curses library for + # the terminfo package, as such we'll need to make sure we only look in the system path for the curses library + # and not pick up the tinfo from the nix provided ncurses package. + # + # We also need to force us to use the systems COREFOUNDATION, not the one that nix builds. Again this is impure, + # but it will allow us to have proper binary distributions. + # + # do not use nixpkgs provided core foundation + export NIX_COREFOUNDATION_RPATH=/System/Library/Frameworks + # drop curses from the LDFLAGS, we really want the system ones, not the nix ones. + export NIX_LDFLAGS=$(for lib in $NIX_LDFLAGS; do case "$lib" in *curses*);; *) echo -n "$lib ";; esac; done;) + export NIX_CFLAGS_COMPILE+=" -Wno-nullability-completeness -Wno-availability -Wno-expansion-to-defined -Wno-builtin-requires-header -Wno-unused-command-line-argument" + + # unconditionally add the MacOSX.sdk and TargetConditional.h + export NIX_CFLAGS_COMPILE+=" -isystem /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include" + export NIX_LDFLAGS="-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib $NIX_LDFLAGS" + + ''; + + nativeBuildInputs = (with pkgs; [ + # This needs to come *before* ghc, + # otherwise we migth end up with the clang from + # the bootstrap GHC in PATH with higher priority. + clang_11 + llvm_11 + + haskell.compiler.${compiler} + haskell.packages.${compiler}.cabal-install + haskell.packages.${compiler}.alex + haskell.packages.${compiler}.happy # _1_19_12 is needed for older GHCs. + + automake + autoconf + m4 + + gmp + zlib.out + zlib.dev + glibcLocales + # locale doesn't build yet :-/ + # locale + + git + + python3 + # python3Full + # python3Packages.sphinx + perl + + which + wget + curl + file + + xz + xlibs.lndir + + cacert ]) + ++ (with pkgs.darwin.apple_sdk.frameworks; [ Foundation Security ]); +} diff --git a/cabal.project b/cabal.project index d56c8d2189..bc5f59487c 100644 --- a/cabal.project +++ b/cabal.project @@ -25,6 +25,8 @@ packages: ./plugins/hls-ormolu-plugin ./plugins/hls-call-hierarchy-plugin +optional-packages: vendored/*/*.cabal + tests: true package * From 336d032aad9b24043ed60c1ee72e209642f2bd05 Mon Sep 17 00:00:00 2001 From: Javier Neira Date: Sat, 18 Sep 2021 14:56:37 +0200 Subject: [PATCH 2/4] Add comment --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21773896a9..b52ea3e9ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,5 @@ +# Used for ci setup in the gitlab mirror of the project: +# https://gitlab.haskell.org/haskell/haskell-language-server/-/pipelines variables: # Commit of ghc/ci-images repository from which to pull Docker images DOCKER_REV: "853f348f9caf38b08740b280296fbd34e09abb3a" From 0c67040afd23d9f2b651c1f3fcd638c63da547d0 Mon Sep 17 00:00:00 2001 From: Javier Neira Date: Sat, 18 Sep 2021 15:00:24 +0200 Subject: [PATCH 3/4] Comment about vendored packages --- cabal.project | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cabal.project b/cabal.project index f475cae127..f245368276 100644 --- a/cabal.project +++ b/cabal.project @@ -25,6 +25,8 @@ packages: ./plugins/hls-ormolu-plugin ./plugins/hls-call-hierarchy-plugin +-- Standard location for temporary packages needed for particular environments +-- For example it is used in the project gitlab mirror to help in the MAcOS M1 build script optional-packages: vendored/*/*.cabal tests: true From 3ff4f191eef3fa02a4b6ab1d399f5cfef176641f Mon Sep 17 00:00:00 2001 From: Javier Neira Date: Sat, 18 Sep 2021 15:05:07 +0200 Subject: [PATCH 4/4] Add link to gitlab ci config --- cabal.project | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cabal.project b/cabal.project index f245368276..50d1ffbbd1 100644 --- a/cabal.project +++ b/cabal.project @@ -26,7 +26,8 @@ packages: ./plugins/hls-call-hierarchy-plugin -- Standard location for temporary packages needed for particular environments --- For example it is used in the project gitlab mirror to help in the MAcOS M1 build script +-- For example it is used in the project gitlab mirror to help in the MAcOS M1 build script +-- See https://github.com/haskell/haskell-language-server/blob/master/.gitlab-ci.yml optional-packages: vendored/*/*.cabal tests: true