Zstd compression for sort files

.github/workflows/deploy.yml

@@ -75,7 +75,7 @@ jobs:
         if: runner.os == 'Linux'
         uses: awalsh128/cache-apt-pkgs-action@v1
         with:
-          packages: libmpfr-dev
+          packages: libmpfr-dev libzstd-dev
           version: 1.0
 
       - name: Install dependencies (Windows)
@@ -103,7 +103,7 @@ jobs:
 
       - name: Configure
         run: |
-          opts='--disable-dependency-tracking --disable-scalar --disable-threaded --disable-native --enable-static-link --with-gmp --with-zlib'
+          opts='--disable-dependency-tracking --disable-scalar --disable-threaded --disable-native --enable-static-link --with-gmp --with-zlib --with-zstd'
           case ${{ matrix.bin }} in
             form)  opts="$opts --enable-scalar";;
             tform) opts="$opts --enable-threaded";;
@@ -113,19 +113,22 @@ jobs:
           # -l:libgmp.a to make partial static links possible.
           # As a workaround, we make a library directory with libgmp.a
           # but without libgmp.dylib so that the linker has to link libgmp.a.
+          # The same for other libraries.
           # Note that the Homebrew installation path for Apple Silicon (arm64)
           # differs from the one on macOS Intel (x86-64).
-          mkdir static-lib
           if [ "$RUNNER_OS" == "macOS" ]; then
+            mkdir static-lib
             if [ "$RUNNER_ARCH" == "ARM64" ]; then
               ln -s /opt/homebrew/opt/gmp/lib/libgmp.a static-lib/libgmp.a
               ln -s /opt/homebrew/opt/mpfr/lib/libmpfr.a static-lib/libmpfr.a
-              # The GMP and MPFR include directories, not located in the usual places,
+              ln -s /opt/homebrew/opt/zstd/lib/libzstd.a static-lib/libzstd.a
+              # Include directories, not located in the usual places,
               # must be explicitly appended to the include paths.
-              export CPATH="/opt/homebrew/opt/gmp/include:/opt/homebrew/opt/mpfr/include:${CPATH:-}"
+              export CPATH="/opt/homebrew/opt/gmp/include:/opt/homebrew/opt/mpfr/include:/opt/homebrew/opt/zstd/include:${CPATH:-}"
             else
               ln -s /usr/local/opt/gmp/lib/libgmp.a static-lib/libgmp.a
               ln -s /usr/local/opt/mpfr/lib/libmpfr.a static-lib/libmpfr.a
+              ln -s /usr/local/opt/zstd/lib/libzstd.a static-lib/libzstd.a
             fi
             export LIBRARY_PATH="$(pwd)/static-lib:${LIBRARY_PATH:-}"
             opts="$opts --disable-static-link"
@@ -141,7 +144,7 @@ jobs:
         run: |
           if [ "$RUNNER_OS" == "macOS" ]; then
             if [ "$RUNNER_ARCH" == "ARM64" ]; then
-              export CPATH="/opt/homebrew/opt/gmp/include:/opt/homebrew/opt/mpfr/include:${CPATH:-}"
+              export CPATH="/opt/homebrew/opt/gmp/include:/opt/homebrew/opt/mpfr/include:/opt/homebrew/opt/zstd/include:${CPATH:-}"
             fi
             export LIBRARY_PATH="$(pwd)/static-lib:${LIBRARY_PATH:-}"
           fi

.github/workflows/test.yml

@@ -31,10 +31,10 @@ jobs:
         with:
           fetch-depth: 0  # ensures a reachable tag
 
-      - name: Install MPFR
+      - name: Install dependencies
         uses: awalsh128/cache-apt-pkgs-action@v1
         with:
-          packages: libmpfr-dev
+          packages: libmpfr-dev libzstd-dev
           version: 1.0
 
       - name: Install MPI if necessary
@@ -54,7 +54,7 @@ jobs:
           case ${{ matrix.bin }} in
             vorm|tvorm|parvorm) opts="$opts --enable-debug";;
           esac
-          opts="$opts --with-gmp --with-zlib"
+          opts="$opts --with-gmp --with-zlib --with-zstd"
           autoreconf -i
           ./configure $opts
 
@@ -138,16 +138,10 @@ jobs:
         with:
           fetch-depth: 0  # ensures a reachable tag
 
-      - name: Install Valgrind
-        uses: awalsh128/cache-apt-pkgs-action@v1
-        with:
-          packages: valgrind
-          version: 1.0
-
-      - name: Install MPFR
+      - name: Install dependencies
         uses: awalsh128/cache-apt-pkgs-action@v1
         with:
-          packages: libmpfr-dev
+          packages: libmpfr-dev libzstd-dev valgrind
           version: 1.0
 
       - name: Install MPI if necessary
@@ -167,7 +161,7 @@ jobs:
           case ${{ matrix.bin }} in
             vorm|tvorm|parvorm) opts="$opts --enable-debug";;
           esac
-          opts="$opts --with-gmp --with-zlib"
+          opts="$opts --with-gmp --with-zlib --with-zstd"
           autoreconf -i
           ./configure $opts
 
@@ -196,16 +190,10 @@ jobs:
         with:
           fetch-depth: 0  # ensures a reachable tag
 
-      - name: Install LCOV
-        uses: awalsh128/cache-apt-pkgs-action@v1
-        with:
-          packages: lcov
-          version: 1.0
-
-      - name: Install MPFR
+      - name: Install dependencies
         uses: awalsh128/cache-apt-pkgs-action@v1
         with:
-          packages: libmpfr-dev
+          packages: lcov libmpfr-dev libzstd-dev
           version: 1.0
 
       - name: Install MPI if necessary
@@ -222,7 +210,7 @@ jobs:
             tvorm)   opts="$opts --disable-scalar --enable-threaded --disable-parform";;
             parvorm) opts="$opts --disable-scalar --disable-threaded --enable-parform";;
           esac
-          opts="$opts --enable-debug --enable-coverage --with-gmp --with-zlib"
+          opts="$opts --enable-debug --enable-coverage --with-gmp --with-zlib --with-zstd"
           autoreconf -i
           ./configure $opts
 
@@ -275,7 +263,12 @@ jobs:
       - name: Install dependencies
         run: |
           apt-get update
-          apt-get -y install automake build-essential git libgmp-dev libmpfr-dev ruby zlib1g-dev
+          apt-get -y install automake build-essential git libgmp-dev libmpfr-dev libzstd-dev ruby zlib1g-dev
+
+      # Fix dubious ownership for Git operations.
+      # See https://github.com/actions/runner/issues/2033#issuecomment-1204205989
+      - name: Set ownership
+        run: chown -R $(id -u):$(id -g) $PWD
 
       - name: Configure
         run: |
@@ -284,7 +277,7 @@ jobs:
             form)  opts="$opts --enable-scalar --disable-threaded --disable-parform";;
             tform) opts="$opts --disable-scalar --enable-threaded --disable-parform";;
           esac
-          opts="$opts --disable-debug --with-gmp --with-zlib"
+          opts="$opts --disable-debug --with-gmp --with-zlib --with-zstd"
           autoreconf -i
           ./configure $opts
 

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "extern/zstd"]
+	path = extern/zstd
+	url = https://github.com/facebook/zstd

configure.ac

@@ -66,14 +66,20 @@ m4_define([serial_tests], [m4_esyscmd_s([
 	${AUTOMAKE:-automake} --version | head -1 |
 	awk '{split ($NF,a,"."); if (a[1] >= 2 || (a[1] == 1 && a[2] >= 13)) { print "serial-tests" }}'
 ])])
+# If automake >= 1.14, use the subdir-objects option of AM_INIT_AUTOMAKE (available in automake >= 1.13)
+# to suppress the "possible forward-incompatibility" warning.
+m4_define([subdir_objects], [m4_esyscmd_s([
+	${AUTOMAKE:-automake} --version | head -1 |
+	awk '{split ($NF,a,"."); if (a[1] >= 2 || (a[1] == 1 && a[2] >= 14)) { print "subdir-objects" }}'
+])])
 
 AC_PREREQ([2.59])
 AC_INIT([FORM], [FORM_VERSION], [https://github.com/vermaseren/form/issues])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([sources/form3.h])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([build-aux])
-AM_INIT_AUTOMAKE([1.7 foreign -Wall dist-bzip2] serial_tests)
+AM_INIT_AUTOMAKE([1.7 foreign -Wall dist-bzip2] serial_tests subdir_objects)
 
 # Check for .version file
 AM_CONDITIONAL([FIXED_VERSION], [test -f $srcdir/.version])
@@ -350,6 +356,40 @@ AS_IF([test "x$with_zlib" != xno],
 		AC_MSG_NOTICE([zlib is not available])
 		with_zlib=no])])
 
+# Check for zstd
+AC_ARG_WITH([zstd],
+	[AS_HELP_STRING([--with-zstd@<:@=DIR@:>@],
+		[use zstd for compression (installed in prefix DIR) @<:@default=check@:>@])],
+	[AS_IF([test "x$withval" != xyes && test "x$withval" != xno && test "x$withval" != xcheck],
+		[with_zstd=yes
+		CPPFLAGS="$CPPFLAGS -I$withval/include"
+		LDFLAGS="$LDFLAGS -L$withval/lib"])],
+	[with_zstd=check])
+AS_IF([test "x$with_zstd" != xno],
+	[flag=:
+	AS_IF([$flag], [AS_IF([test "x$with_zlib" = xno],
+		[AC_MSG_NOTICE([using zstd requires zlib])
+		flag=false])])
+	AS_IF([$flag], [AC_CHECK_HEADER([zstd.h], [], [flag=false])])
+	AS_IF([$flag], [AC_CHECK_LIB([zstd], [ZSTD_versionNumber], [LIBS="-lzstd $LIBS"], [flag=false])])
+	AS_IF([$flag],
+		[AC_DEFINE(WITHZSTD, [], [Define to use zstd for compression.])
+		with_zstd=yes],
+		[AS_IF([test "x$with_zstd" = xyes],
+			[AC_MSG_FAILURE([test for zstd failed. Give --without-zstd if you want to compile without zstd])])
+		AC_MSG_NOTICE([zstd is not available])
+		with_zstd=no])])
+AM_CONDITIONAL([WITHZSTD], [test "x$with_zstd" = xyes])
+
+# Ensure zstd/zlibWrapper
+AS_IF([test -f "$srcdir/extern/zstd/zlibWrapper/zstd_zlibwrapper.h"], [],
+	[AC_PATH_PROG([GIT], [git])
+		AS_IF([test -n "$GIT"], [(cd "$srcdir" && "$GIT" submodule update --init)])])
+AS_IF([test -f "$srcdir/extern/zstd/zlibWrapper/zstd_zlibwrapper.h"], [],
+	[AS_IF([test -d "$srcdir/.git"],
+		[AC_MSG_FAILURE([$srcdir/extern/zstd/zlibWrapper does not exist. Run git submodule update --init in the repository directory])],
+		[AC_MSG_FAILURE([$srcdir/extern/zstd/zlibWrapper does not exist])])])
+
 # enable-scalar/threaded/parform/debug/float
 AC_ARG_ENABLE([scalar],
 	[AS_HELP_STRING([--enable-scalar],
@@ -1063,6 +1103,10 @@ if test "x$with_zlib" = xyes; then
 	echo "    zlib"
 	atleastone=yes
 fi
+if test "x$with_zstd" = xyes; then
+	echo "    zstd"
+	atleastone=yes
+fi
 if test $atleastone = no; then
 	echo "    <NONE>"
 fi

doc/manual/statements.tex

@@ -3903,14 +3903,28 @@ \section{on}
 \rightvitem{13cm}{Turns compression mode on. This compression is a 
 relatively simple compression that hardly costs extra computer time but 
 saves roughly a factor two in disk storage. The old statement was `compress 
-on' but this should be avoided in the future. This setting is the default.}
+on' but this should be avoided in the future. This setting is the default if
+\FORM\ has not been compiled with gzip or zstd support.}
 
 \leftvitem{3.5cm}{compress,gzip\index{gzip}}
-\rightvitem{13cm}{This option should be followed by a comma or a space and 
+\rightvitem{13cm}{This setting is the default, if \FORM\ has been compiled
+with gzip support but not with zstd support.
+This option may be followed by a comma or a space and 
 a single digit. It activates the gzip compression for the sort file. This 
 compression can make the intermediate sort file considerably shorter at the 
-cost of some CPU time. This option can be used when disk space is at a 
-premium. The digit indicates the compression level. Zero means no 
+cost of some CPU time.
+The digit indicates the compression level. Zero means no 
+compression and 9 is the highest level. The default level is 6. Above that 
+the compression becomes very slow and doesn't gain very much extra.}
+
+\leftvitem{3.5cm}{compress,zstd\index{zstd}}
+\rightvitem{13cm}{This setting is the default, if \FORM\ has been compiled with
+zstd support.
+This option may be followed by a comma or a space and 
+a single digit. It activates the zstd compression for the sort file. This 
+compression performs better than gzip, both in terms of compression/
+decompression rates and compression ratio.
+The digit indicates the compression level. Zero means no 
 compression and 9 is the highest level. The default level is 6. Above that 
 the compression becomes very slow and doesn't gain very much extra.}
 

sources/Makefile.am

@@ -71,6 +71,8 @@ SRCBASE = \
 	vector.h \
 	wildcard.c
 
+INCBASE =
+
 if ONUNIX
 SRCBASE += \
 	unixfile.c \
@@ -88,6 +90,19 @@ SRCBASE += \
 	float.c
 endif
 
+if WITHZSTD
+SRCBASE += \
+	$(top_srcdir)/extern/zstd/zlibWrapper/gzclose.c \
+	$(top_srcdir)/extern/zstd/zlibWrapper/gzcompatibility.h \
+	$(top_srcdir)/extern/zstd/zlibWrapper/gzguts.h \
+	$(top_srcdir)/extern/zstd/zlibWrapper/gzlib.c \
+	$(top_srcdir)/extern/zstd/zlibWrapper/gzread.c \
+	$(top_srcdir)/extern/zstd/zlibWrapper/gzwrite.c \
+	$(top_srcdir)/extern/zstd/zlibWrapper/zstd_zlibwrapper.c \
+	$(top_srcdir)/extern/zstd/zlibWrapper/zstd_zlibwrapper.h
+INCBASE += -I$(top_srcdir)/extern/zstd/zlibWrapper
+endif
+
 SRCPTHREAD = \
 	threads.c
 
@@ -143,7 +158,7 @@ bin_PROGRAMS =
 if BUILD_FORM
 bin_PROGRAMS += form
 form_SOURCES = $(SRCBASE)
-form_CPPFLAGS =
+form_CPPFLAGS = $(INCBASE)
 form_CFLAGS = $(COMPILEFLAGS)
 form_CXXFLAGS = $(COMPILEFLAGS)
 form_LDFLAGS = $(LINKFLAGS) $(STATIC_LDFLAGS)
@@ -158,7 +173,7 @@ endif
 if BUILD_VORM
 bin_PROGRAMS += vorm
 vorm_SOURCES = $(SRCBASE)
-vorm_CPPFLAGS = -DDEBUGGING
+vorm_CPPFLAGS = -DDEBUGGING $(INCBASE)
 vorm_CFLAGS = $(DEBUGCOMPILEFLAGS)
 vorm_CXXFLAGS = $(DEBUGCOMPILEFLAGS)
 vorm_LDFLAGS = $(DEBUGLINKFLAGS)
@@ -173,7 +188,7 @@ endif
 if BUILD_TFORM
 bin_PROGRAMS += tform
 tform_SOURCES = $(SRCBASE) $(SRCPTHREAD)
-tform_CPPFLAGS = -DWITHPTHREADS $(PTHREAD_CPPFLAGS)
+tform_CPPFLAGS = -DWITHPTHREADS $(PTHREAD_CPPFLAGS) $(INCBASE)
 tform_CFLAGS = $(COMPILEFLAGS) $(PTHREAD_CFLAGS)
 tform_CXXFLAGS = $(COMPILEFLAGS) $(PTHREAD_CFLAGS)
 tform_LDFLAGS = $(LINKFLAGS) $(STATIC_LDFLAGS)
@@ -188,7 +203,7 @@ endif
 if BUILD_TVORM
 bin_PROGRAMS += tvorm
 tvorm_SOURCES = $(SRCBASE) $(SRCPTHREAD)
-tvorm_CPPFLAGS = -DWITHPTHREADS -DDEBUGGING $(PTHREAD_CPPFLAGS)
+tvorm_CPPFLAGS = -DWITHPTHREADS -DDEBUGGING $(PTHREAD_CPPFLAGS) $(INCBASE)
 tvorm_CFLAGS = $(DEBUGCOMPILEFLAGS) $(PTHREAD_CFLAGS)
 tvorm_CXXFLAGS = $(DEBUGCOMPILEFLAGS) $(PTHREAD_CFLAGS)
 tvorm_LDFLAGS = $(DEBUGLINKFLAGS)
@@ -203,7 +218,7 @@ endif
 if BUILD_PARFORM
 bin_PROGRAMS += parform
 parform_SOURCES = $(SRCBASE) $(SRCPARALLEL)
-parform_CPPFLAGS = -DWITHMPI -DPF_WITHGETENV -DPF_WITHLOG $(MPI_CPPFLAGS)
+parform_CPPFLAGS = -DWITHMPI -DPF_WITHGETENV -DPF_WITHLOG $(MPI_CPPFLAGS) $(INCBASE)
 parform_CFLAGS = $(COMPILEFLAGS) $(MPI_CFLAGS)
 parform_CXXFLAGS = $(COMPILEFLAGS) $(MPI_CXXFLAGS)
 parform_LDFLAGS = $(LINKFLAGS) $(MPI_STATIC_LDFLAGS)
@@ -218,7 +233,7 @@ endif
 if BUILD_PARVORM
 bin_PROGRAMS += parvorm
 parvorm_SOURCES = $(SRCBASE) $(SRCPARALLEL)
-parvorm_CPPFLAGS = -DWITHMPI -DPF_WITHGETENV -DPF_WITHLOG -DDEBUGGING $(MPI_CPPFLAGS)
+parvorm_CPPFLAGS = -DWITHMPI -DPF_WITHGETENV -DPF_WITHLOG -DDEBUGGING $(MPI_CPPFLAGS) $(INCBASE)
 parvorm_CFLAGS = $(DEBUGCOMPILEFLAGS) $(MPI_CFLAGS)
 parvorm_CXXFLAGS = $(DEBUGCOMPILEFLAGS) $(MPI_CXXFLAGS)
 parvorm_LDFLAGS = $(DEBUGLINKFLAGS)