Merge branch 'release/8.6.0' into main

.github/workflows/api-change.yml

@@ -6,7 +6,7 @@ on:
       tag1:
         description: 'First ESMF Tag'
         required: true
-        default: 'v8.3.0'
+        default: 'v8.5.0'
       tag2:
         description: 'Second ESMF Tag'
         required: true

.github/workflows/build-esmf-docs.yml

@@ -46,8 +46,8 @@ jobs:
       uses: actions/checkout@v3
       with:
           repository: esmf-org/esmf-org.github.io
-          path: esmf-org.github.io
-          token: ${{ secrets.ESMF_WEB_TOKEN }}
+          path: ${{github.workspace}}/esmf-org.github.io
+          ssh-key: ${{secrets.ESMF_WEB_DEPLOY_KEY}}
 
     - name: Copy docs
       run: | 
@@ -66,10 +66,10 @@ jobs:
         cp -rf ./dev_guide/dev_guide/* ${{ github.workspace }}/esmf-org.github.io/docs/nightly/${{ github.ref_name }}/dev_guide/
         
     - name: Commit and publish docs
-      uses: actions-js/push@master
-      with:
-        repository: esmf-org/esmf-org.github.io
-        directory: ${{ github.workspace }}/esmf-org.github.io
-        branch: master
-        github_token: ${{ secrets.ESMF_WEB_TOKEN }}
-        message: 'Publish ESMF Docs'
+      run: | 
+        cd ${{ github.workspace }}/esmf-org.github.io
+        git config user.name "github-actions[bot]"
+        git config user.email "github-actions[bot]@users.noreply.github.com"
+        git add .
+        git diff-index --quiet HEAD || git commit --message "Publish ESMF Docs"
+        git push origin

.github/workflows/build-esmpy-docs.yml

@@ -41,13 +41,13 @@ jobs:
       with:
         name: esmpy-docs
         path: ${{ github.workspace }}/artifacts
-        
+
     - name: Checkout esmpy_doc
       uses: actions/checkout@v3
       with:
           repository: esmf-org/esmpy_doc
-          path: esmpy_doc
-          token: ${{ secrets.ESMF_WEB_TOKEN }}
+          path: ${{github.workspace}}/esmpy_doc
+          ssh-key: ${{secrets.ESMPY_WEB_DEPLOY_KEY}}
 
     - name: Copy docs
       run: | 
@@ -60,10 +60,10 @@ jobs:
         cp -rf latex/ESMPy.pdf ${{ github.workspace }}/esmpy_doc/docs/nightly/${{ github.ref_name }}/
 
     - name: Commit and publish docs
-      uses: actions-js/push@master
-      with:
-        repository: esmf-org/esmpy_doc
-        directory: ${{ github.workspace }}/esmpy_doc
-        branch: master
-        github_token: ${{ secrets.ESMF_WEB_TOKEN }}
-        message: 'Publish ESMPy Docs'
+      run: |
+        cd ${{ github.workspace }}/esmpy_doc
+        git config user.name "github-actions[bot]"
+        git config user.email "github-actions[bot]@users.noreply.github.com"
+        git add .
+        git diff-index --quiet HEAD || git commit --message "Publish ESMPy Docs"
+        git push origin

.github/workflows/scripts/spack_concretize.sh

@@ -134,6 +134,29 @@ if [[ "$comp" == *"oneapi"* ]]; then
   echo "::endgroup::"
 fi
 
+# check given gcc compiler is found or not? If not, use newer version
+if [[ "$comp" == *"gcc"* ]]; then
+  echo "::group::Check gcc compiler"
+  comp_str=${comp/@/@=}
+  str=`echo $comp_str | awk -F\@ '{print $1}'`
+  comp_ver=`grep -ir "${str}@=" ~/.spack/linux/compilers.yaml | tr -d "spec: ${str}@=" | sort -n | tail -n 1`
+
+  use_latest=0
+  if [[ "$comp" == *"gcc@latest"* ]]; then
+     echo "The gcc@latest is set. Trying to find latest available gcc compiler ..."
+     use_latest=1
+  elif [ -z "$(cat ~/.spack/linux/compilers.yaml | grep $comp_str)" ]; then
+     echo "Given compiler ($comp) is not found! Exiting ..."
+     exit 1
+  fi
+
+  if [[ $use_latest == 1 ]]; then
+     comp="${str}@$comp_ver"
+  fi
+  echo "Using $comp gnu compiler."
+  echo "::endgroup::"
+fi
+
 # create spack.yaml
 echo "::group::Create spack.yaml"
 echo "spack:" > spack.yaml

.github/workflows/test-build-spack.yml

@@ -1,13 +1,18 @@
 name: Build ESMF Using Spack
 
 env:
-  compiler: gcc@11.3.0
+  compiler: gcc@latest
   esmf_version: 'esmf@develop'
   nuopc_app_version: 'develop'
 
 on:
   workflow_dispatch:
     inputs:
+      compiler:
+        description: 'Compiler version like gcc@12.3.0 or gcc@latest (latest available version on runner)'
+        required: false
+        type: string
+        default: 'gcc@latest'
       esmf_version: 
         description: 'ESMF version or tag like esmf@develop or esmf@=8.5.0b23'
         required: false
@@ -39,7 +44,11 @@ jobs:
         echo "ENV  : >${{ env.esmf_version }}< >${{ env.nuopc_app_version }}<"
 
         # create matrix JSON file
-        str1="{\"compiler\": [\"${{ env.compiler }}\"],"
+        if [ -z "${{ inputs.compiler }}" ]; then
+          str1="{\"compiler\": [\"${{ env.compiler }}\"],"
+        else
+          str1="{\"compiler\": [\"${{ inputs.compiler }}\"],"  
+        fi
         if [ -z "${{ inputs.esmf_version }}" ]; then
           str2="\"esmf\": [\"${{ env.esmf_version }}+external-parallelio\", \"${{ env.esmf_version }}~external-parallelio\"]}"
           echo "nuopc_app_version=${{ env.nuopc_app_version }}" >> $GITHUB_OUTPUT

.gitignore

@@ -12,6 +12,7 @@ PET*.ESMF_LogFile
 # esmf build generated directories #
 ####################################
 /DEFAULTINSTALLDIR/
+/apps/
 /lib/
 /mod/
 /obj/

README.md

@@ -32,7 +32,7 @@ Pre-built binaries for ESMF and ESMPy are available through a number of channels
    ```
    docker run -it --rm esmf/esmf-build-release:latest
    ```
-   Replace `latest` in the above command with a valid version, like `8.5.0`, in order to access a specific ESMF version.
+   Replace `latest` in the above command with a valid version, like `8.6.0`, in order to access a specific ESMF version.
 
  * [Anaconda Conda-Forge](https://anaconda.org/conda-forge/): Under [conda-forge/esmpy](https://anaconda.org/conda-forge/esmpy). To install locally (_note Windows is not supported_), run:
    ```

build/common.mk

@@ -215,6 +215,10 @@ ifndef ESMF_TRACE_LIB_BUILD
 export ESMF_TRACE_LIB_BUILD = default
 endif
 
+ifndef ESMF_TRACE_PRELOAD_LINKED
+export ESMF_TRACE_PRELOAD_LINKED = default
+endif
+
 ifndef ESMF_FORTRANSYMBOLS
 export ESMF_FORTRANSYMBOLS = default
 endif
@@ -445,6 +449,10 @@ ifneq ($(ESMF_TRACE_LIB_BUILD),OFF)
 export ESMF_TRACE_LIB_BUILD = ON
 endif
 
+ifneq ($(ESMF_TRACE_PRELOAD_LINKED),ON)
+export ESMF_TRACE_PRELOAD_LINKED = OFF
+endif
+
 ifneq ($(ESMF_TESTCOMPTUNNEL),OFF)
 export ESMF_TESTCOMPTUNNEL = ON
 endif
@@ -987,6 +995,7 @@ ESMF_F90LINKLIBS = $(ESMF_F90LINKLIBS_ENV)
 endif
 ESMF_F90LINKLIBS     +=
 ESMF_F90ESMFLINKLIBS += -lesmf $(ESMF_F90LINKLIBS)
+ESMF_F90ESMFPRELOADLINKLIBS += -lesmf $(ESMF_TRACE_DYNAMICLINKLIBS) $(ESMF_F90LINKLIBS)
 
 # - CXXLINKER
 ifneq ($(origin ESMF_CXXLINKER), environment)
@@ -1794,6 +1803,38 @@ ESMF_F90LINKRPATHSTHIRD   += $(ESMF_F90RPATHPREFIX)$(ESMF_BABELTRACE_LIBPATH)
 endif
 endif
 
+#-------------------------------------------------------------------------------
+# NVML
+#-------------------------------------------------------------------------------
+ifeq ($(ESMF_NVML),ON)
+ESMF_NVML = standard
+endif
+ifeq ($(ESMF_NVML),standard)
+ifneq ($(origin ESMF_NVML_LIBS), environment)
+ESMF_NVML_LIBS = -lnvidia-ml
+endif
+endif
+
+ifdef ESMF_NVML
+ESMF_CPPFLAGS                += -DESMF_NVML=1
+ifdef ESMF_NVML_INCLUDE
+ESMF_CXXCOMPILEPATHSTHIRD    += -I$(ESMF_NVML_INCLUDE)
+ESMF_F90COMPILEPATHSTHIRD    += -I$(ESMF_NVML_INCLUDE)
+endif
+ifdef ESMF_NVML_LIBS
+ESMF_CXXLINKLIBS          += $(ESMF_NVML_LIBS)
+ESMF_CXXLINKRPATHSTHIRD   += $(addprefix $(ESMF_CXXRPATHPREFIX),$(subst -L,,$(filter -L%,$(ESMF_NVML_LIBS))))
+ESMF_F90LINKLIBS          += $(ESMF_NVML_LIBS)
+ESMF_F90LINKRPATHSTHIRD   += $(addprefix $(ESMF_F90RPATHPREFIX),$(subst -L,,$(filter -L%,$(ESMF_NVML_LIBS))))
+endif
+ifdef ESMF_NVML_LIBPATH
+ESMF_CXXLINKPATHSTHIRD    += -L$(ESMF_NVML_LIBPATH)
+ESMF_F90LINKPATHSTHIRD    += -L$(ESMF_NVML_LIBPATH)
+ESMF_CXXLINKRPATHSTHIRD   += $(ESMF_CXXRPATHPREFIX)$(ESMF_NVML_LIBPATH)
+ESMF_F90LINKRPATHSTHIRD   += $(ESMF_F90RPATHPREFIX)$(ESMF_NVML_LIBPATH)
+endif
+endif
+
 #-------------------------------------------------------------------------------
 # Set the correct MPIRUN command with appropriate options
 #-------------------------------------------------------------------------------
@@ -1820,7 +1861,11 @@ ifeq ($(ESMF_OPENMP),OFF)
 ESMF_CPPFLAGS       += -DESMF_NO_OPENMP
 endif
 
-ifeq ($(ESMF_OPENMP),ON)
+ifeq ($(ESMF_OPENMP),OMP4)
+ESMF_CPPFLAGS       += -DESMF_OPENMP4
+endif
+
+ifneq ($(ESMF_OPENMP),OFF)
 ESMF_F90COMPILEOPTS += $(ESMF_OPENMP_F90COMPILEOPTS)
 ESMF_F90LINKOPTS    += $(ESMF_OPENMP_F90LINKOPTS)
 ESMF_CXXCOMPILEOPTS += $(ESMF_OPENMP_CXXCOMPILEOPTS)
@@ -1986,32 +2031,52 @@ ifeq ($(ESMF_TRACE_BUILD_SHARED),ON)
 ESMF_TRACE_LDPRELOAD := $(ESMF_LIBDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)
 ESMF_PRELOADSCRIPT = $(ESMF_LIBDIR)/preload.sh
 
-ifneq ($(ESMF_OS),Darwin)
-ESMF_ENV_PRELOAD = LD_PRELOAD
+ESMF_SL_PRELOAD_LIBLINKER = $(ESMF_CXXCOMPILER)
+
+ifeq ($(ESMF_OS),Darwin)
+ESMF_ENV_PRELOAD          = DYLD_INSERT_LIBRARIES
+ESMF_ENV_PRELOAD_DELIMIT  = ':'
+ifeq ($(ESMF_COMM),openmpi)
+# make sure to link in the Fortran MPI bindings
+ESMF_SL_PRELOAD_LIBLINKER = $(ESMF_F90COMPILER)
+endif
 else
-ESMF_ENV_PRELOAD = DYLD_INSERT_LIBRARIES
+ESMF_ENV_PRELOAD          = LD_PRELOAD
+ESMF_ENV_PRELOAD_DELIMIT  = ' '
 endif
 
 # MPI implementations do not pick up LD_PRELOAD
 # so we pass a small script to each MPI task
 ifneq (,$(findstring mpich,$(ESMF_COMM)))
 ESMF_PRELOAD_SH = $(ESMF_PRELOADSCRIPT)
 endif
+ifeq ($(ESMF_COMM),openmpi)
+ESMF_PRELOAD_SH = $(ESMF_PRELOADSCRIPT)
+endif
 ifeq ($(ESMF_COMM),mpi)
 ESMF_PRELOAD_SH = $(ESMF_PRELOADSCRIPT)
 endif
 ifeq ($(ESMF_COMM),mpt)
 ESMF_PRELOAD_SH = $(ESMF_PRELOADSCRIPT)
 endif
+ifneq (,$(findstring srun,$(ESMF_MPIRUN)))
+ESMF_PRELOAD_SH = $(ESMF_PRELOADSCRIPT)
+endif
 
 endif
 
 build_preload_script:
 	-@echo "#!/bin/sh" > $(ESMF_PRELOADDIR)/preload.sh
 	-@echo "# Script to preload ESMF dynamic trace library" >> $(ESMF_PRELOADDIR)/preload.sh
-	-@echo 'env LD_PRELOAD="$$LD_PRELOAD $(ESMF_PRELOADDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)" $$*' >> $(ESMF_PRELOADDIR)/preload.sh
+	-@echo 'if [ "$$$(ESMF_ENV_PRELOAD)" != "" ]; then' >> $(ESMF_PRELOADDIR)/preload.sh
+	-@echo 'env $(ESMF_ENV_PRELOAD)="$$$(ESMF_ENV_PRELOAD)$(ESMF_ENV_PRELOAD_DELIMIT)$(ESMF_PRELOADDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)" $$*' >> $(ESMF_PRELOADDIR)/preload.sh
+	-@echo 'else' >> $(ESMF_PRELOADDIR)/preload.sh
+	-@echo 'env $(ESMF_ENV_PRELOAD)="$(ESMF_PRELOADDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)" $$*' >> $(ESMF_PRELOADDIR)/preload.sh
+	-@echo 'fi' >> $(ESMF_PRELOADDIR)/preload.sh
 	chmod 755 $(ESMF_PRELOADDIR)/preload.sh
 
+ESMF_TRACE_DYNAMICLINKLIBS := -lesmftrace_preload
+
 ESMF_TRACE_STATICLINKLIBS := -lesmftrace_static
 
 ESMF_TRACE_WRAPPERS_IO  := write writev pwrite read open

build_config/Cygwin.g95.default/build_rules.mk

@@ -34,8 +34,6 @@ ESMF_F90LINKLIBS       += -lpmpich -lmpich
 ESMF_CXXDEFAULT         = mpiCC
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),mpich2)
 # Mpich2 ---------------------------------------------------
@@ -44,8 +42,6 @@ ESMF_CXXDEFAULT         = mpicxx
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
 ESMF_MPIMPMDRUNDEFAULT  = mpiexec $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),mpich)
 # Mpich3 and up --------------------------------------------
@@ -63,8 +59,6 @@ ESMF_CXXDEFAULT         = mpic++
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
 ESMF_MPIMPMDRUNDEFAULT  = mpiexec $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),openmpi)
 # OpenMPI --------------------------------------------------

build_config/Cygwin.gfortran.default/build_rules.mk

@@ -34,8 +34,6 @@ ESMF_F90LINKLIBS       += -lpmpich -lmpich
 ESMF_CXXDEFAULT         = mpiCC
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),mpich2)
 # Mpich2 ---------------------------------------------------
@@ -44,8 +42,6 @@ ESMF_CXXDEFAULT         = mpicxx
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
 ESMF_MPIMPMDRUNDEFAULT  = mpiexec $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),mpich)
 # Mpich3 and up --------------------------------------------
@@ -63,8 +59,6 @@ ESMF_CXXDEFAULT         = mpic++
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
 ESMF_MPIMPMDRUNDEFAULT  = mpiexec $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),msmpi)
 # Microsofts version of MPICH2 on CCS 2003 is generally at:

build_config/Darwin.absoft.default/build_rules.mk

@@ -33,8 +33,6 @@ ESMF_F90DEFAULT         = mpif90
 ESMF_CXXDEFAULT         = mpiCC
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),mpich2)
 # Mpich2 ---------------------------------------------------
@@ -43,8 +41,6 @@ ESMF_CXXDEFAULT         = mpicxx
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
 ESMF_MPIMPMDRUNDEFAULT  = mpiexec $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),mpich)
 # Mpich3 and up --------------------------------------------
@@ -71,8 +67,6 @@ ESMF_CXXDEFAULT         = mpic++
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
 ESMF_MPIMPMDRUNDEFAULT  = mpiexec $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),openmpi)
 # OpenMPI --------------------------------------------------

build_config/Darwin.g95.default/build_rules.mk

@@ -34,8 +34,6 @@ ESMF_F90LINKLIBS       += -lpmpich++ -lmpich
 ESMF_CXXDEFAULT         = mpicxx
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),mpich2)
 # Mpich2 ---------------------------------------------------
@@ -44,8 +42,6 @@ ESMF_CXXDEFAULT         = mpicxx
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
 ESMF_MPIMPMDRUNDEFAULT  = mpiexec $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),mpich)
 # Mpich3 and up --------------------------------------------
@@ -72,8 +68,6 @@ ESMF_CXXDEFAULT         = mpic++
 ESMF_CDEFAULT           = mpicc
 ESMF_MPIRUNDEFAULT      = mpirun $(ESMF_MPILAUNCHOPTIONS)
 ESMF_MPIMPMDRUNDEFAULT  = mpiexec $(ESMF_MPILAUNCHOPTIONS)
-ESMF_F90COMPILECPPFLAGS+= -DESMF_NO_MPI3
-ESMF_CXXCOMPILECPPFLAGS+= -DESMF_NO_MPI3
 else
 ifeq ($(ESMF_COMM),openmpi)
 # OpenMPI --------------------------------------------------