Free disk space in container and prepare building the CLI (#267)

* Upgrade version of checkout action

* Query disk-space and largest packages

* Free disk space when building the container

* Remove more packages from image

* Install `cargo-c` with `--locked` to lower MSRV

* Install `zlib-static` into container

* Add missing `-y` parameter

* Disable some features to debug linking problems

* Print compiler information inside container

* Try static relocation model

* Try `-no-pie` instead

* Try both flags

* Try with `-Clink-args=-pie`

* Remove `-Clink-args=-pie` and only build static libraries

* Compile packages in container with `--with-pic=no`

* Re-enable dynamic libraries and PIC

* Build with `-Crelocation-model=pic` in container

* Build CLI only to locate the build failures

* Add missing `--locked` switch

* Do not use `--features=static` for CI tests

* Try compiling the CLI with `-Crelocation-model=dynamic-no-pic`

* Re-enable nightly Rust in the container

* Compile dependencies in container always with PIC

* Build the CLI with all features

* Compile zlib manually with PIC in the container

* Remove building CLI in Rust workflow

.github/workflows/container.yml

@@ -11,7 +11,27 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: Free disk space
+        run: |
+          # inspired by: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
+          df -h
+          # 100 largest packages, in ascending order
+          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
+          sudo apt-get remove -y google-cloud-cli
+          sudo apt-get remove -y azure-cli
+          sudo apt-get remove -y microsoft-edge-stable
+          sudo apt-get remove -y '^dotnet-.*'
+          sudo apt-get remove -y '^temurin-.*-jdk'
+          sudo apt-get remove -y google-chrome-stable
+          sudo apt-get remove -y '^llvm-.*-dev'
+          sudo apt-get remove -y firefox
+          sudo apt-get remove -y powershell
+          sudo apt-get remove -y mono-devel
+          sudo apt-get autoremove -y
+          sudo apt-get clean
+          # disk space after removing packages
+          df -h
       - name: Build image
         uses: redhat-actions/buildah-build@v2
         with:

.github/workflows/rust.yml

@@ -9,7 +9,6 @@ defaults:
 
 env:
   CARGO_TERM_COLOR: always
-  RUSTFLAGS: '-Cinstrument-coverage'
 
 jobs:
   build:
@@ -61,9 +60,11 @@ jobs:
           echo "RUSTDOCFLAGS=-Cinstrument-coverage -Z unstable-options --persist-doctests $(pwd)/target/debug/doctestbins" >> "$GITHUB_ENV"
 
       - name: Run tests
+        env:
+          RUSTFLAGS: '-Cinstrument-coverage'
         run: |
           # we need stderr, but we can't run test twice because it'll regenerate/modify the binaries which interferes with `llvm-cov`
-          cargo test --all-features --no-fail-fast 2> >(tee stderr 1>&2)
+          cargo test --features=applgrid,evolve,fastnlo,fktable --no-fail-fast 2> >(tee stderr 1>&2)
           # from https://stackoverflow.com/a/51141872/812178
           sed -i 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' stderr
 

maintainer/pineappl-ci/Containerfile

@@ -5,9 +5,11 @@ ARG APPLGRID_V=1.6.27
 ARG CARGOC_V=0.9.24+cargo-0.73.0
 ARG FASTNLO_V=2.5.0-2826
 ARG LHAPDF_V=6.4.0
+ARG ZLIB_V=1.3.1
 
 # the last version is the default Rust version used in the container
-ARG RUST_V="1.70.0"
+# as long as we're using `persist-doctests` in the `Rust` workflow we need nightly as default
+ARG RUST_V="1.70.0 nightly-2024-01-25"
 
 ENV APPL_IGRID_DIR="/usr/local/src/applgrid-${APPLGRID_V}/src"
 ENV CARGO_HOME="/usr/local/cargo"

maintainer/pineappl-ci/script.sh

@@ -2,6 +2,13 @@
 
 set -euo pipefail
 
+# print this so we can see whether the compiler/linker has `--enable-default-pie` enabled; if it's
+# not enabled we need to build our dependencies with `--with-pic=yes` (see below)
+echo "--- COMPILER/LINKER INFORMATION"
+echo "int main() {}" > test.c
+cc -Q -v test.c
+echo "---"
+
 # install rustup
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 
@@ -20,8 +27,11 @@ done
 # needed by the vendored OpenSSL used by `cargo-c`
 yum -y install perl-IPC-Cmd
 
+# needed by `pineappl_applgrid` with `--features=static`
+yum -y install zlib-static
+
 # install cargo-c needed for the CAPI
-cargo install cargo-c --version ${CARGOC_V} --features=vendored-openssl
+cargo install --locked cargo-c --version ${CARGOC_V} --features=vendored-openssl
 
 # remove files generated by cargo
 rm -r /usr/local/cargo/registry
@@ -30,36 +40,46 @@ rm -r /usr/local/cargo/registry
 ( curl "https://lhapdf.hepforge.org/downloads/?f=LHAPDF-${LHAPDF_V}.tar.gz" || \
     curl "https://web.archive.org/web/20211018095814/https://lhapdf.hepforge.org/downloads/?f=LHAPDF-6.4.0.tar.gz" ) | tar xzf -
 cd LHAPDF-${LHAPDF_V}
-./configure --disable-python
-make -j
+# compile static libraries with PIC to make statically linking PineAPPL's CLI work
+# see also https://users.rust-lang.org/t/why-does-crelocation-model-dynamic-no-pic-help-although-it-shouldnt/109012
+./configure --disable-python --with-pic=yes
+make -j V=1
 make install
 ldconfig
-
 cd ..
 
 # install PDF sets
 for pdf in NNPDF31_nlo_as_0118_luxqed NNPDF40_nnlo_as_01180 NNPDF40_nlo_as_01180; do
     curl "https://lhapdfsets.web.cern.ch/current/${pdf}.tar.gz" | tar xzf - -C /usr/local/share/LHAPDF
 done
 
+# install zlib compiled with `-fPIC`
+curl "https://www.zlib.net/zlib-${ZLIB_V}.tar.gz" | tar xzf -
+cd zlib-${ZLIB_V}
+CFLAGS=-fPIC ./configure --prefix=/usr/local
+make -j
+make install
+ldconfig
+cd ..
+
 # install APPLgrid
 curl "https://applgrid.hepforge.org/downloads?f=applgrid-${APPLGRID_V}.tgz" | tar xzf -
 cd applgrid-${APPLGRID_V}
-./configure --without-root
+# compile static libraries with PIC to make statically linking PineAPPL's CLI work
+./configure --without-root --with-pic=yes
 make -j
 make install
 ldconfig
 mkdir -p ${APPL_IGRID_DIR}
 cp src/*.h ${APPL_IGRID_DIR}
-
 cd ..
 
 # install fastNLO
 curl "https://fastnlo.hepforge.org/code/v25/fastnlo_toolkit-${FASTNLO_V}.tar.gz" | tar xzf -
 cd fastnlo_toolkit-${FASTNLO_V}
-./configure --prefix=/usr/local/
-make -j
+# compile static libraries with PIC to make statically linking PineAPPL's CLI work
+./configure --prefix=/usr/local/ --with-pic=yes
+make -j V=1
 make install
 ldconfig
-
 cd ..