v7 Branch: Update to GFE v1.16.0, ESMF 8.6.1, NCO 5.2.6 (#217)

* Update to GFE v1.16.0 * Update to ESMF 8.6.1 * Update to curl 8.8.0 * Update to NCO 5.2.6 * Update VERSION * Backport patches from v8 Baselibs
GEOS-ESM · Jul 11, 2024 · 17a48e2 · 17a48e2
1 parent 484504f
commit 17a48e2
Show file tree

Hide file tree

Showing 13 changed files with 2,160 additions and 59 deletions.
diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
@@ -1,14 +1,15 @@
 name: Build Baselibs
 on:
   release:
-    types: 
+    types:
       - published
+  workflow_dispatch:
 
 jobs:
   build_baselibs:
     runs-on: ubuntu-latest
-    container: 
-      image: gmao/ubuntu20-openmpi:4.0.5-gcc_10.2.0
+    container:
+      image: gmao/ubuntu20-openmpi:5.0.2-gcc_13.2.0
       credentials:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -20,10 +21,14 @@ jobs:
       CC: gcc
       CXX: g++
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: recursive
+      - name: Set all directories as git safe
+        run: |
+          git config --global --add safe.directory '*'
       - name: Versions etc.
         run: |
           gcc --version

diff --git a/Base.mk b/Base.mk
@@ -377,17 +377,9 @@ endif
 
   ifeq ($(ESMF_COMM),mpt)
     MPICC := mpicc
-    ifeq ($(findstring ifort,$(notdir $(FC))),ifort)
-      CC := gcc
-      CC_FROM_ENV := FALSE
-    endif
   endif
   ifeq ($(ESMF_COMM),mpt)
     MPICXX := mpicxx
-    ifeq ($(findstring ifort,$(notdir $(FC))),ifort)
-      CXX := g++
-      CXX_FROM_ENV := FALSE
-    endif
   endif
 
 # Make sure we have compilers

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,36 @@
 ### Removed
 ### Added
 
+## [7.25.0] - 2024-07-11
+
+### Updates
+
+- ESMF v8.6.1
+- GFE v1.16.0
+  - gFTL v1.14.0
+  - gFTL-shared v1.9.0
+  - fArgParse v1.8.0
+  - pFUnit v4.10.0
+  - yaFyaml v1.4.0
+- curl 8.8.0
+- NCO 5.2.6
+
+### Fixed
+
+- Fixes for MPT and using icx/icpx at NAS
+- The updated `config.guess` file requires an updated `config.sub` file. This is now copied in the `configure` step for all libraries that have explicit `config.guess` copies
+- Fix rules in `esmf_rules.mk` where in `ESMPy` is now `esmpy`
+- Changed the HDF5 build issue code to detect if XCode is 15 or greater. The `-Wl,-ld_classic` flag is needed (at the moment) only for the HDF5 build on macOS with Clang 15 or greater.
+
+### Changed
+
+- Added `autoreconf -f -v -i` to many config steps. This is needed for newer hardware (like Graviton3 and GraceHopper)
+  - For `antlr2` and `SDPToolkit` we instead copy a new `config.guess` as `autoreconf` didn't seem to work
+- Add new `SYSTEM_ZLIB` variable which, if set to `YES`, does not build zlib and uses the system version.
+  This was shown to be needed in testing NVHPC on bucy. Not sure why yet as no other compiler cares.
+  - Manifested as a `/usr/lib64/libxml2.so: undefined reference to `gzopen64@ZLIB_1.2.3.3'` error in ESMF linking
+- Update docker image in CI workflow
+
 ## [7.24.1] - 2024-05-15
 
 ### Fixed

diff --git a/GFE b/GFE
diff --git a/GNUmakefile b/GNUmakefile
@@ -346,6 +346,24 @@ ifeq ($(findstring ifx,$(notdir $(FC))),ifx)
    SUBDIRS := $(filter-out SDPToolkit,$(SUBDIRS))
 endif
 
+# NVHPC and ESMF seem to have issues with zlib in Baselibs
+# So we need to filter it out. We will use a special
+# build variable to do this
+ifeq ('$(SYSTEM_ZLIB)','YES')
+   SUBDIRS := $(filter-out zlib,$(SUBDIRS))
+   ZLIB_INSTALL =
+   WITH_ZLIB =
+   WITH_ZLIB_SHORT =
+   CURL_ZLIB = --without-zlib
+   # We also need to filter out zlib from INC_SUPP
+   INC_SUPP := $(filter-out -I$(prefix)/include/zlib,$(INC_SUPP))
+else
+   ZLIB_INSTALL = zlib.install
+   WITH_ZLIB := --with-zlib=$(prefix)/include/zlib,$(prefix)/lib
+   WITH_ZLIB_SHORT := --with-zlib=$(prefix)
+   CURL_ZLIB := $(WITH_ZLIB_SHORT)
+endif
+
 TARGETS = all lib install
 
 download: gsl.download szlib.download cdo.download hdfeos.download hdfeos5.download SDPToolkit.download
@@ -369,6 +387,12 @@ verify:
 	@echo CC_IS_CLANG = $(CC_IS_CLANG)
 	@echo NO_IMPLICIT_FUNCTION_ERROR = $(NO_IMPLICIT_FUNCTION_ERROR)
 	@echo LIB_EXTRA = $(LIB_EXTRA)
+	@echo SYSTEM_ZLIB = $(SYSTEM_ZLIB)
+	@echo ZLIB_INSTALL = $(ZLIB_INSTALL)
+	@echo WITH_ZLIB = $(WITH_ZLIB)
+	@echo WITH_ZLIB_SHORT = $(WITH_ZLIB_SHORT)
+	@echo CURL_ZLIB = $(CURL_ZLIB)
+	@echo INC_SUPP = $(INC_SUPP)
 	@echo NAG_FCFLAGS = $(NAG_FCFLAGS)
 	@echo FC_FROM_ENV = $(FC_FROM_ENV)
 	@echo CC_FROM_ENV = $(CC_FROM_ENV)
@@ -531,13 +555,14 @@ jpeg.config: jpeg/configure
 	  export PATH="$(prefix)/bin:$(PATH)" ;\
 	  export CPPFLAGS="$(INC_SUPP)";\
 	  export LIBS="-lm";\
+	  autoreconf -f -v -i ;\
 	  ./configure --prefix=$(prefix) \
 		      --includedir=$(prefix)/include/jpeg \
 		      --disable-shared \
 		      CFLAGS="$(CFLAGS)" CC=$(CC) CXX=$(CXX) FC=$(FC) )
 	@touch $@
 
-hdf4.config: hdf4/README.md jpeg.install zlib.install szlib.install
+hdf4.config: hdf4/README.md jpeg.install $(ZLIB_INSTALL) szlib.install
 	@echo Configuring hdf4
 	@(cd hdf4; \
           export PATH="$(prefix)/bin:$(PATH)" ;\
@@ -549,24 +574,25 @@ hdf4.config: hdf4/README.md jpeg.install zlib.install szlib.install
                       --includedir=$(prefix)/include/hdf \
                       --with-jpeg=$(prefix)/include/jpeg,$(prefix)/lib \
                       --with-szlib=$(prefix)/include/szlib,$(prefix)/lib \
-                      --with-zlib=$(prefix)/include/zlib,$(prefix)/lib \
+                      $(WITH_ZLIB) \
                       --disable-netcdf \
                       --enable-hdf4-xdr \
                       $(HDF4_ENABLE_FORTRAN) \
                       CFLAGS="$(CFLAGS) $(NO_IMPLICIT_FUNCTION_ERROR) $(NO_IMPLICIT_INT_ERROR)" FFLAGS="$(NAG_FCFLAGS) $(NAG_DUSTY) $(ALLOW_ARGUMENT_MISMATCH)" CC=$(CC) FC=$(FC) CXX=$(CXX) )
 	touch $@
 
-hdf5.config :: hdf5/README.md szlib.install zlib.install
+hdf5.config :: hdf5/README.md szlib.install $(ZLIB_INSTALL)
 	echo Configuring hdf5
 	(cd hdf5; \
           export PATH="$(prefix)/bin:$(PATH)" ;\
           export LIBS="-lm" ;\
           export LDFLAGS="$(CLANG_LD_CLASSIC)" ;\
+          autoreconf -f -v -i;\
           ./configure --prefix=$(prefix) \
                       --includedir=$(prefix)/include/hdf5 \
                       --with-szlib=$(prefix)/include/szlib,$(prefix)/lib \
-                      --with-zlib=$(prefix)/include/zlib,$(prefix)/lib \
-                      --disable-shared --disable-cxx\
+                      $(WITH_ZLIB) \
+                      --disable-shared --disable-cxx \
                       --enable-hl --enable-fortran --disable-sharedlib-rpath \
                       $(ENABLE_GPFS) $(H5_PARALLEL) $(HDF5_ENABLE_F2003) \
                       CFLAGS="$(CFLAGS) $(HDF5_NCCS_MPT_CFLAG)" FCFLAGS="$(NAG_FCFLAGS)" CC=$(NC_CC) FC=$(NC_FC) CXX=$(NC_CXX) F77=$(NC_F77) )
@@ -589,6 +615,7 @@ netcdf.config : netcdf/configure
           export CPPFLAGS="$(CPPFLAGS) $(NC_CPPFLAGS) $(INC_SUPP)";\
           export CFLAGS="$(CFLAGS) $(NC_CFLAGS) $(PTHREAD_FLAG)";\
           export LIBS="-L$(prefix)/lib -lsz -ljpeg $(LINK_GPFS) $(LIB_CURL) -ldl -lm $(LIB_EXTRA)" ;\
+          autoreconf -f -v -i;\
           ./configure --prefix=$(prefix) \
                       --includedir=$(prefix)/include/netcdf \
                       $(ENABLE_HDF4) \
@@ -611,6 +638,7 @@ netcdf-fortran.config : netcdf-fortran/configure netcdf.install
           export CPPFLAGS="$(CPPFLAGS) -I$(prefix)/include/netcdf $(INC_SUPP)";\
           export CFLAGS="$(CFLAGS) $(NC_CFLAGS) $(PTHREAD_FLAG)";\
           export LIBS="-L$(prefix)/lib $(LIB_NETCDF) $(LIB_CURL)" ;\
+          autoreconf -f -v -i;\
           ./configure --prefix=$(prefix) \
                       --includedir=$(prefix)/include/netcdf \
                       $(NC_PAR_TESTS) \
@@ -627,6 +655,7 @@ netcdf-cxx4.config : netcdf-cxx4/configure netcdf.install
           export PATH="$(prefix)/bin:$(PATH)" ;\
           export CPPFLAGS="$(CPPFLAGS) -I$(prefix)/include/netcdf $(INC_SUPP)";\
           export LIBS="-L$(prefix)/lib $(LIB_NETCDF) $(LIB_CURL)" ;\
+          autoreconf -f -v -i;\
           ../configure --prefix=$(prefix) \
                        --includedir=$(prefix)/include/netcdf \
                        --disable-shared \
@@ -666,6 +695,7 @@ nco.config : nco/configure
           export CXXFLAGS="$(NCO_CXXFLAGS)";\
           export CFLAGS="$(CFLAGS) $(PTHREAD_FLAG)";\
           export LIBS="-L$(prefix)/lib $(LIB_NETCDF) $(LIB_HDF5) $(LIB_HDF4) -lsz  -ljpeg $(LINK_GPFS) $(LIB_CURL) -ldl -lm $(LIB_EXTRA)" ;\
+          autoreconf -f -v -i;\
           ./configure --prefix=$(prefix) \
                       --includedir=$(prefix)/include/nco \
                       --enable-ncoxx \
@@ -690,6 +720,7 @@ szlib.config : szlib.download szlib/configure
           export PATH="$(prefix)/bin:$(PATH)" ;\
           export CPPFLAGS="$(INC_SUPP)";\
           export LIBS="-lm";\
+          autoreconf -f -v -i;\
           ./configure --prefix=$(prefix) \
                       --includedir=$(prefix)/include/szlib \
                       --disable-shared \
@@ -706,7 +737,7 @@ zlib.config : zlib/configure
 	touch $@
 
 
-curl.config : curl/configure.ac zlib.install
+curl.config : curl/configure.ac $(ZLIB_INSTALL)
 	@echo "Configuring curl"
 	@(cd curl; \
           export PATH="$(prefix)/bin:$(PATH)" ;\
@@ -716,7 +747,7 @@ curl.config : curl/configure.ac zlib.install
           ./configure --prefix=$(prefix) \
                       --includedir=$(prefix)/include/ \
                       --libdir=$(prefix)/lib \
-                      --with-zlib=$(prefix) \
+                      $(WITH_ZLIB_SHORT) \
                       --disable-ldap \
                       --enable-manual \
                       --disable-shared \
@@ -741,10 +772,11 @@ cdo.config: cdo.download cdo/configure netcdf.install udunits2.install
           export PATH="$(prefix)/bin:$(PATH)" ;\
           export CPPFLAGS="$(CPPFLAGS) $(INC_SUPP)";\
           export LIBS="-L$(prefix)/lib $(LIB_NETCDF) $(LIB_CURL) -lexpat $(LIB_HDF4) -lsz -ljpeg $(LINK_GPFS) -ldl -lm" ;\
+          autoreconf -f -v -i;\
           ./configure --prefix=$(prefix) \
                       --includedir=$(prefix)/include/cdo \
                       --with-szlib=$(prefix) \
-                      --with-zlib=$(prefix) \
+                      $(WITH_ZLIB_SHORT) \
                       --with-hdf5=$(prefix) \
                       --with-netcdf=$(prefix) \
                       --with-udunits2=$(prefix) \
@@ -779,12 +811,15 @@ GFE.config:
 	@echo "Configuring GFE"
 	@mkdir -p ./GFE/build
 	@(cd ./GFE; \
+		export LDFLAGS="$(CLANG_LD_CLASSIC)" ;\
 		cmake -B build -S . --install-prefix=$(prefix) -DCMAKE_PREFIX_PATH=$(prefix) -DSKIP_OPENMP=YES )
 	@touch $@
 
 antlr2.config : antlr2/configure
 	@echo "Configuring antlr2"
 	@mkdir -p ./antlr2/build
+	@cp config.guess ./antlr2/scripts/config.guess
+	@cp config.sub ./antlr2/scripts/config.sub
 	@(cd antlr2/build; \
           export PATH="$(prefix)/bin:$(PATH)" ;\
           export CPPFLAGS="$(INC_SUPP)";\
@@ -809,6 +844,7 @@ gsl.config : gsl.download gsl/configure
           export PATH="$(prefix)/bin:$(PATH)" ;\
           export CPPFLAGS="$(INC_SUPP)";\
           export LIBS="-lm";\
+          autoreconf -f -v -i;\
           ./configure --prefix=$(prefix) \
                       --includedir=$(prefix)/include/gsl \
                       --disable-shared \
@@ -852,6 +888,7 @@ hdfeos5.config: hdfeos5.download hdfeos5/configure hdf5.install
           export PATH="$(prefix)/bin:$(PATH)" ;\
           export CPPFLAGS="$(CPPFLAGS) $(INC_SUPP)";\
           export LIBS="-L$(prefix)/lib $(LIB_NETCDF) $(LIB_CURL) -lexpat $(LIB_HDF4) -lsz -ljpeg $(LINK_GPFS) -ldl -lm" ;\
+          autoreconf -f -v -i;\
           ./configure --prefix=$(prefix) \
                       --includedir=$(prefix)/include/hdfeos5 \
                       --disable-shared --enable-static \
@@ -870,13 +907,15 @@ SDPToolkit.download : scripts/download_SDPToolkit.bash
 
 SDPToolkit.config: SDPToolkit.download SDPToolkit/configure hdfeos5.install
 	@echo "Configuring SDPToolkit $*"
+	@cp config.guess ./TOOLKIT/config/config.guess
+	@cp config.sub ./TOOLKIT/config/config.sub
 	@(cd SDPToolkit; \
           export PATH="$(prefix)/bin:$(PATH)" ;\
           export CPPFLAGS="$(CPPFLAGS) $(INC_SUPP_SDP)";\
           export LIBS="-L$(prefix)/lib $(LIB_NETCDF) $(LIB_CURL) -lexpat $(LIB_HDF4) -lsz -ljpeg $(LINK_GPFS) -ldl -lm" ;\
           ./configure --prefix=$(prefix) \
                       --includedir=$(prefix)/include/SDPToolkit \
-                      --with-zlib=$(prefix)/include/zlib,$(prefix)/lib \
+                      $(WITH_ZLIB) \
                       --with-szlib=$(prefix)/include/szlib,$(prefix)/lib \
                       --with-hdf4=$(prefix)/include/hdf,$(prefix)/lib \
                       --with-hdf5=$(prefix)/include/hdf5,$(prefix)/lib \

diff --git a/README.md b/README.md
@@ -12,22 +12,22 @@ NASA/GSFC.
 
 | Library                                                                  | Version      |
 | ---                                                                      | ---          |
-| [ESMF](https://github.com/esmf-org/esmf)                                 | v8.6.1b04    |
+| [ESMF](https://github.com/esmf-org/esmf)                                 | v8.6.1       |
 | [netCDF](https://github.com/Unidata/netcdf-c)                            | 4.9.2        |
 | [netCDF Fortran](https://github.com/Unidata/netcdf-fortran)              | 4.6.1        |
 | [netCDF C++](https://github.com/Unidata/netcdf-cxx4)                     | 4.3.1        |
 | [HDF5](https://portal.hdfgroup.org/display/support)                      | 1.10.11      |
 | [HDF4](https://portal.hdfgroup.org/display/support)                      | 4.2.16-2     |
-| [GFE](https://github.com/Goddard-Fortran-Ecosystem/GFE)                  | v1.15.0      |
+| [GFE](https://github.com/Goddard-Fortran-Ecosystem/GFE)                  | v1.16.0      |
 | [xgboost](https://github.com/dmlc/xgboost)                               | v1.6.0       |
 | [antlr2](https://www.antlr2.org/)                                        | 2.7.7        |
 | [GSL](https://www.gnu.org/software/gsl/)                                 | 2.7          |
 | [jpeg](http://www.ijg.org/)                                              | 9e           |
 | [zlib](http://www.zlib.net/)                                             | 1.3.1        |
 | [szip](https://support.hdfgroup.org/doc_resource/SZIP/)                  | 2.1.1        |
-| [cURL](https://curl.haxx.se/)                                            | 8.7.1        |
+| [cURL](https://curl.haxx.se/)                                            | 8.8.0        |
 | [UDUNITS2](https://github.com/GMAO-SI-Team/UDUNITS-2.git)                | 2.2.28       |
-| [NCO](http://nco.sourceforge.net/)                                       | 5.2.4        |
+| [NCO](http://nco.sourceforge.net/)                                       | 5.2.6        |
 | [CDO](https://code.mpimet.mpg.de/projects/cdo)                           | 2.3.0        |
 | [nccmp](https://gitlab.com/remikz/nccmp)                                 | 1.9.1.0      |
 | [HDF-EOS2](https://wiki.earthdata.nasa.gov/display/DAS)                  | 3.0          |

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-7.24.1
+7.25.0
+55 −40		.github/workflows/main.yml
+18 −0		ChangeLog.md
+5 −5		README.md
+1 −1		fArgParse
+1 −1		gFTL
+1 −1		gFTL-shared
+1 −1		pFUnit
+1 −1		tools/ci-install-mpi.sh
+1 −1		yaFyaml