pkgcheck.sh: verify make and cmake install same bits; tests madler#497 …

…and madler#499

Run cmake first and configure/make second, see
issue madler#499 (build breaks due to zconf.h missing)

ci: add fedora, debian; use matrix and a template in .cirrus.yml

.cirrus.yml

@@ -1,55 +1,64 @@
-check_ubu1804_task:
-  container:
-    dockerfile: tooling/cirrus/ubu1804/Dockerfile
-  test_script:
-    - cc --version
-    - mkdir btmp; cd btmp; cmake -G Ninja .. && ninja && ctest -V
-  always:
-    junit_artifacts:
-      path: "btmp/*-results.xml"
-      type: text/xml
-      format: junit
+upload_junit_template: &UPLOAD_JUNIT
+   always:
+     junit_artifacts:
+       path: "btmp/*-results.xml"
+       type: text/xml
+       format: junit
 
-check_ubu2004_task:
-  container:
-    dockerfile: tooling/cirrus/ubu2004/Dockerfile
+check_docker_task:
+  matrix:
+    - name: "ubu1404"
+      container:
+        dockerfile: tooling/cirrus/ubu1404/Dockerfile   # remove 2022/4
+    - name: "ubu1604"
+      container:
+        dockerfile: tooling/cirrus/ubu1604/Dockerfile   # remove 2024/4
+    - name: "ubu1804"
+      container:
+        dockerfile: tooling/cirrus/ubu1804/Dockerfile   # remove 2028/4
+    - name: "ubu2004"
+      container:
+        dockerfile: tooling/cirrus/ubu2004/Dockerfile
+    - name: "ubu2010"
+      container:
+        dockerfile: tooling/cirrus/ubu2010/Dockerfile   # remove 2021/7
+    - name: "debian8"
+      container:
+        dockerfile: tooling/cirrus/debian8/Dockerfile   # remove 2020/7
+    - name: "debian9"
+      container:
+        dockerfile: tooling/cirrus/debian9/Dockerfile   # remove 2022/7
+    - name: "debian10"
+      container:
+        dockerfile: tooling/cirrus/debian10/Dockerfile
+    - name: "fedora31"
+      container:
+        dockerfile: tooling/cirrus/fedora31/Dockerfile  # remove 2020/11
+    - name: "fedora32"
+      container:
+        dockerfile: tooling/cirrus/fedora32/Dockerfile  # remove 2021/5
   test_script:
-    - cc --version
-    - mkdir btmp; cd btmp; cmake -G Ninja .. && ninja && ctest -V
-  always:
-    junit_artifacts:
-      path: "btmp/*-results.xml"
-      type: text/xml
-      format: junit
+    - cat /etc/issue || true; cat /etc/os-release || true; uname -a || true
+    - sh tooling/test-cmake.sh && sh tooling/pkgcheck.sh
+  << : *UPLOAD_JUNIT
 
 check_mac_task:
   osx_instance:
     image: catalina-xcode
   install_script:
     - brew install ninja
   test_script:
-    - cc --version
-    - ninja --version
-    - mkdir btmp; cd btmp; cmake -G Ninja .. && cmake --build . && ctest -V
-  always:
-    junit_artifacts:
-      path: "btmp/*-results.xml"
-      type: text/xml
-      format: junit
+    - sh tooling/test-cmake.sh && sh tooling/pkgcheck.sh
+  << : *UPLOAD_JUNIT
 
 check_bsd_task:
   freebsd_instance:
     image_family: freebsd-12-1
   install_script:
     - pkg install -y gcc cmake ninja
   test_script:
-    - cc --version
-    - mkdir btmp; cd btmp; cmake -G Ninja .. && ninja && ctest -V
-  always:
-    junit_artifacts:
-      path: "btmp/*-results.xml"
-      type: text/xml
-      format: junit
+    - sh tooling/test-cmake.sh && sh tooling/pkgcheck.sh
+  << : *UPLOAD_JUNIT
 
 check_win_task:
   windows_container:
@@ -61,8 +70,4 @@ check_win_task:
     - ninja --version
     - echo "Setting compiler to MSVC rather than the default (GCC from chocolatey)"
     - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat" && where cl && set CC=cl && mkdir btmp && cd btmp && cmake -G Ninja .. && ninja && ctest -V
-  always:
-    junit_artifacts:
-      path: "btmp/*-results.xml"
-      type: text/xml
-      format: junit
+  << : *UPLOAD_JUNIT

tooling/cirrus/debian10/Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:10
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y cmake gcc libc6-dev ninja-build make && \
+    apt-get clean

tooling/cirrus/debian8/Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:8
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y cmake gcc libc6-dev ninja-build make && \
+    apt-get clean

tooling/cirrus/debian9/Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:9
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y cmake gcc libc6-dev ninja-build make && \
+    apt-get clean

tooling/cirrus/fedora31/Dockerfile

@@ -0,0 +1,12 @@
+FROM fedora:31
+
+RUN dnf -y update && dnf clean all
+
+RUN dnf makecache \
+  && dnf -y install \
+    gcc \
+    cmake \
+    diffutils \
+    make \
+    ninja-build \
+  && dnf clean all

tooling/cirrus/fedora32/Dockerfile

@@ -0,0 +1,12 @@
+FROM fedora:32
+
+RUN dnf -y update && dnf clean all
+
+RUN dnf makecache \
+  && dnf -y install \
+    gcc \
+    cmake \
+    diffutils \
+    make \
+    ninja-build \
+  && dnf clean all

tooling/cirrus/ubu1404/Dockerfile

@@ -0,0 +1,5 @@
+FROM ubuntu:14.04
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y cmake gcc libc6-dev ninja-build make && \
+    apt-get clean

tooling/cirrus/ubu1604/Dockerfile

@@ -0,0 +1,5 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y cmake gcc libc6-dev ninja-build make && \
+    apt-get clean

tooling/cirrus/ubu1804/Dockerfile

@@ -1,5 +1,5 @@
 FROM ubuntu:18.04
 
 RUN apt-get update && \
-    apt-get install --no-install-recommends -y cmake gcc libc6-dev ninja-build && \
+    apt-get install --no-install-recommends -y cmake gcc libc6-dev ninja-build make && \
     apt-get clean

tooling/cirrus/ubu2004/Dockerfile

@@ -1,5 +1,5 @@
 FROM ubuntu:20.04
 
 RUN apt-get update && \
-    apt-get install --no-install-recommends -y cmake gcc libc6-dev ninja-build && \
+    apt-get install --no-install-recommends -y cmake gcc libc6-dev ninja-build make && \
     apt-get clean

tooling/cirrus/ubu2010/Dockerfile

@@ -0,0 +1,5 @@
+FROM ubuntu:20.10
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y cmake gcc libc6-dev ninja-build make && \
+    apt-get clean

tooling/pkgcheck.sh

@@ -0,0 +1,103 @@
+#!/bin/sh
+# Verify that the various build systems produce identical results on a Unixlike system
+set -ex
+
+# Tell GNU's ld etc. to use Jan 1 1970 when embedding timestamps
+export SOURCE_DATE_EPOCH=0
+# Tell Apple's ar etc. to use zero timestamps
+export ZERO_AR_DATE=1
+
+# New build system
+# Happens to delete top-level zconf.h
+# (which itself is a bug, https://github.com/madler/zlib/issues/162 )
+# which triggers another bug later in configure,
+# https://github.com/madler/zlib/issues/499
+rm -rf btmp2 pkgtmp2
+mkdir btmp2 pkgtmp2
+export DESTDIR=$(pwd)/pkgtmp2
+cd btmp2
+  cmake -G Ninja ..
+  ninja -v
+  ninja install
+cd ..
+
+# Original build system
+rm -rf btmp1 pkgtmp1
+mkdir btmp1 pkgtmp1
+export DESTDIR=$(pwd)/pkgtmp1
+cd btmp1
+  case $(uname) in
+  Darwin)
+    # Tell configure which compiler etc. to use to match cmake
+    # Tell configure to pad the executable the same way cmake will
+    CC="$(xcrun --find gcc || echo gcc)" \
+    CFLAGS="-DNDEBUG -O3 -isysroot $(xcrun --show-sdk-path)" \
+    LDFLAGS="-Wl,-headerpad_max_install_names" \
+    ../configure
+    ;;
+  FreeBSD|Linux)
+    # 1) tell configure to tell ar to be deterministic (skipped, see below)
+    # 2) configure fails with cc, so use clang or gcc
+    # 3) Force same CC for configure and clang
+    #export ARFLAGS="Drc"
+    if grep -i fedora /etc/os-release > /dev/null
+    then
+        # Note: Fedora patches cmake to use -O2 in release, which
+        # does not match the -O3 configure sets :-(
+        export CFLAGS="-O2 -DNDEBUG"
+    fi
+    if clang --version
+    then
+        export CC=clang
+        ../configure
+    elif gcc --version
+    then
+        export CC=gcc
+        ../configure
+    else
+        ../configure
+    fi
+    ;;
+  *)
+    ../configure
+    ;;
+  esac
+  make
+  make install
+cd ..
+
+repack_ar() {
+  if ! cmp --silent pkgtmp1/usr/local/lib/libz.a pkgtmp2/usr/local/lib/libz.a
+  then
+    echo "Warning: libz.a does not match.  Assuming ar needs -D option.  Unpacking..."
+    cd pkgtmp1; ar x usr/local/lib/libz.a; rm usr/local/lib/libz.a; cd ..
+    cd pkgtmp2; ar x usr/local/lib/libz.a; rm usr/local/lib/libz.a; cd ..
+  fi
+}
+
+case $(uname) in
+Darwin)
+  # Alas, dylibs still have an embedded hash or something,
+  # so nuke it.
+  # FIXME: find a less fragile way to deal with this.
+  dylib1=$(find pkgtmp1 -name '*.dylib*' -type f)
+  dylib2=$(find pkgtmp2 -name '*.dylib*' -type f)
+  dd conv=notrunc if=/dev/zero of=$dylib1 skip=1337 count=16
+  dd conv=notrunc if=/dev/zero of=$dylib2 skip=1337 count=16
+  ;;
+FreeBSD|Linux)
+  # The ar on newer systems defaults to -D (i.e. deterministic),
+  # but FreeBSD 12.1, Debian 8, and Ubuntu 14.04 seem to not do that.
+  # I had trouble passing -D safely to the ar inside CMakeLists.txt,
+  # so punt and unpack the archive if needed before comparing.
+  repack_ar
+  ;;
+esac
+
+if diff -Nur pkgtmp1 pkgtmp2
+then
+  echo pkgcheck-cmake-bits-identical PASS
+else
+  echo pkgcheck-cmake-bits-identical FAIL
+  exit 1
+fi

tooling/test-cmake.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Build and test with CMake
+set -ex
+
+rm -rf btmp
+mkdir btmp
+cd btmp
+  cmake -G Ninja ..
+  ninja
+  ctest -V
+cd ..

tooling/test-make.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Build and test with Make
+set -ex
+
+rm -rf btmp
+mkdir btmp
+cd btmp1
+  ../configure
+  make
+  make test
+cd ..