Merge branch 'develop' into optimize6

jderber-NOAA · Dec 10, 2023 · f544ed4 · f544ed4
2 parents 603b5c8 + e7b3764
commit f544ed4
Show file tree

Hide file tree

Showing 41 changed files with 4,391 additions and 465 deletions.
diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml
@@ -15,21 +15,21 @@ env:
 
 # The jobs are split into:
 # 1. a dependency build step (setup), and
-# 2. a GSI build step (build)
+# 2. a GSI build step (gsi)
 # The setup is run once and the environment is cached,
-# so each build of GSI can reuse the cached dependencies to save time (and compute).
+# so each subsequent build of GSI can reuse the cached dependencies to save time (and compute).
 
 jobs:
   setup:
     runs-on: ubuntu-latest
 
     steps:
       # Checkout the GSI to get the ci/spack.yaml file
-      - name: checkout-gsi
+      - name: checkout
         if: steps.cache-env.outputs.cache-hit != 'true'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
-            path: GSI
+            path: gsi
 
       # Cache spack, compiler and dependencies
       - name: cache-env
@@ -39,33 +39,34 @@ jobs:
           path: |
             spack
             ~/.spack
-          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }}
+          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }}
 
       # Install dependencies using Spack
       - name: install-dependencies-with-spack
         if: steps.cache-env.outputs.cache-hit != 'true'
         run: |
           sudo apt-get install cmake
-          git clone -c feature.manyFiles=true https://github.com/NOAA-EMC/spack.git
+          git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
-          spack env create gsi-env GSI/ci/spack.yaml
+          spack env create gsi-env gsi/ci/spack.yaml
           spack env activate gsi-env
           spack compiler find
+          sudo apt install cmake
           spack external find
           spack add mpich@3.4.2
           spack concretize
           spack install -v --fail-fast --dirty
           spack clean -a
 
-  build:
+  gsi:
     needs: setup
     runs-on: ubuntu-latest
 
     steps:
-      - name: checkout-gsi
-        uses: actions/checkout@v3
+      - name: checkout
+        uses: actions/checkout@v4
         with:
-            path: GSI
+            path: gsi
 
       - name: cache-env
         id: cache-env
@@ -74,13 +75,13 @@ jobs:
           path: |
             spack
             ~/.spack
-          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }}
+          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }}
 
-      - name: build-gsi
+      - name: build
         run: |
           source spack/share/spack/setup-env.sh
           spack env activate gsi-env
-          cd GSI
+          cd gsi
           mkdir -p build && cd build
           cmake -DCMAKE_INSTALL_PREFIX=../install -DGSI_MODE=Regional -DENKF_MODE=GFS -DBUILD_REG_TESTING=OFF ..
           make -j2 VERBOSE=1

diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml
@@ -18,21 +18,30 @@ env:
 
 # The jobs are split into:
 # 1. a dependency build step (setup), and
-# 2. a GSI build step (build)
+# 2. a GSI build step (gsi)
 # The setup is run once and the environment is cached,
-# so each build of GSI can reuse the cached dependencies to save time (and compute).
+# so each subsequent build of GSI can reuse the cached dependencies to save time (and compute).
 
 jobs:
   setup:
     runs-on: ubuntu-latest
 
     steps:
+      # Free up disk space
+      - name: free-disk-spack
+        run: |
+          df -h
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo apt clean
+          docker rmi $(docker image ls -aq)
+
       # Checkout the GSI to get the ci/spack.yaml file
-      - name: checkout-gsi
+      - name: checkout
         if: steps.cache-env.outputs.cache-hit != 'true'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
-            path: GSI
+            path: gsi
 
       # Cache spack, compiler and dependencies
       - name: cache-env
@@ -43,16 +52,15 @@ jobs:
             spack
             ~/.spack
             /opt/intel
-          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }}
+          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }}
 
       - name: install-intel-compilers
-        if: steps.cache-env.outputs.cache-hit != 'true'
         run: |
           wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
           sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
           echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
           sudo apt-get update
-          sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
+          sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran-2023.2.1 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.1
           sudo apt-get clean
           echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile
 
@@ -62,31 +70,32 @@ jobs:
         run: |
           sudo mv /usr/local/ /usr_local_mv
           sudo apt-get install cmake
-          git clone -c feature.manyFiles=true https://github.com/NOAA-EMC/spack.git
+          git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
-          spack env create gsi-env GSI/ci/spack.yaml
+          spack env create gsi-env gsi/ci/spack.yaml
           spack env activate gsi-env
           spack compiler find
+          sudo apt install cmake
           spack external find
           spack add intel-oneapi-mpi
           spack concretize
           spack install -v --fail-fast --dirty
           spack clean -a
 
-  build:
+  gsi:
     needs: setup
     runs-on: ubuntu-latest
 
     steps:
-      - name: checkout-gsi
-        uses: actions/checkout@v3
-        with:
-            path: GSI
-
       - name: install-intel
         run: |
           echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile
 
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+            path: gsi
+
       - name: cache-env
         id: cache-env
         uses: actions/cache@v3
@@ -95,13 +104,13 @@ jobs:
             spack
             ~/.spack
             /opt/intel
-          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }}
+          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }}
 
-      - name: build-gsi
+      - name: build
         run: |
           source spack/share/spack/setup-env.sh
           spack env activate gsi-env
-          cd GSI
+          cd gsi
           mkdir -p build && cd build
           cmake -DCMAKE_INSTALL_PREFIX=../install -DGSI_MODE=Regional -DENKF_MODE=GFS -DBUILD_REG_TESTING=OFF ..
           make -j2 VERBOSE=1

diff --git a/ci/spack.yaml b/ci/spack.yaml
@@ -6,8 +6,8 @@ spack:
         - intel
         - gcc@10:10
   specs:
-  - netcdf-c@4.7.4
-  - netcdf-fortran@4.5.3
+  - netcdf-c@4.9.2
+  - netcdf-fortran@4.6.0
   - bufr@11.7.0
   - bacio@2.4.1
   - w3emc@2.9.2

diff --git a/modulefiles/gsi_cheyenne.gnu.lua b/modulefiles/gsi_cheyenne.gnu.lua
@@ -1,30 +1,32 @@
 help([[
 ]])
 
-load("cmake/3.22.0")
-load("python/3.7.9")
-load("ncarenv/1.3")
-load("gnu/11.2.0")
-load("mpt/2.25")
-load("ncarcompilers/0.5.0")
-unload("intel")
-unload("netcdf")
 
-prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/modulefiles/stack")
+unload("ncarenv/1.3")
+unload("intel/19.1.1")
+unload("ncarcompilers/0.5.0")
+unload("mpt/2.25")
+unload("netcdf/4.8.1")
 
-load("hpc/1.2.0")
-load("hpc-gnu/11.2.0")
-load("hpc-mpt/2.25")
+prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc")
 
-load("gsi_common")
+local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+local stack_gnu_ver=os.getenv("stack_gnu_ver") or "10.1.0"
+local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.1"
+local cmake_ver=os.getenv("cmake_ver") or "3.22.0"
 
+load(pathJoin("stack-gcc", stack_gnu_ver))
+load(pathJoin("stack-openmpi", stack_openmpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+load(pathJoin("cmake", cmake_ver))
 load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2"))
 load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23"))
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_fix/fix")
+load("gsi_common")
 
-pushenv("CC",  "mpicc")
-pushenv("FC",  "mpif90")
-pushenv("CXX", "mpicxx")
+pushenv("CFLAGS", "-xHOST")
+pushenv("FFLAGS", "-xHOST")
+pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601")
 
 whatis("Description: GSI environment on Cheyenne with GNU Compilers")
diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua
@@ -1,26 +1,32 @@
 help([[
 ]])
 
-load("cmake/3.22.0")
-load("python/3.7.9")
-load("ncarenv/1.3")
-load("intel/2022.1")
-load("mpt/2.25")
-load("ncarcompilers/0.5.0")
+unload("ncarenv/1.3")
+unload("intel/19.1.1")
+unload("ncarcompilers/0.5.0")
+unload("mpt/2.25")
+unload("netcdf/4.8.1")
 
-prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack")
+prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc")
 
-load("hpc/1.2.0")
-load("hpc-intel/2022.1")
-load("hpc-mpt/2.25")
-load("mkl/2022.1")
+local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+local stack_intel_ver=os.getenv("stack_intel_ver") or "19.1.1.217"
+local stack_mpi_ver=os.getenv("stack_mpi_ver") or "2019.7.217"
+local cmake_ver=os.getenv("cmake_ver") or "3.22.0"
 
-load("gsi_common")
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-mpi", stack_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+load(pathJoin("cmake", cmake_ver))
 
-load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2"))
+load("gsi_common")
+load(pathJoin("prod-util", os.getenv("prod_util_ver") or "1.2.2"))
 pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230911")
 
 pushenv("CFLAGS", "-xHOST")
 pushenv("FFLAGS", "-xHOST")
 
+pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601")
+
 whatis("Description: GSI environment on Cheyenne with Intel Compilers")
diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua
@@ -2,22 +2,24 @@ help([[
 Load common modules to build GSI on all machines
 ]])
 
-local netcdf_ver=os.getenv("netcdf_ver") or "4.7.4"
+local netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
+local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0"
 
 local bufr_ver=os.getenv("bufr_ver") or "11.7.0"
 local bacio_ver=os.getenv("bacio_ver") or "2.4.1"
-local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2"
+local w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
 local sp_ver=os.getenv("sp_ver") or "2.3.3"
-local ip_ver=os.getenv("ip_ver") or "3.3.3"
+local ip_ver=os.getenv("ip_ver") or "4.3.0"
 local sigio_ver=os.getenv("sigio_ver") or "2.3.2"
 local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
 local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
 local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0"
 local ncio_ver=os.getenv("ncio_ver") or "1.1.2"
 local crtm_ver=os.getenv("crtm_ver") or "2.4.0"
-local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1"
+local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.2"
 
-load(pathJoin("netcdf", netcdf_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 
 load(pathJoin("bufr", bufr_ver))
 load(pathJoin("bacio", bacio_ver))
@@ -27,7 +29,7 @@ load(pathJoin("ip", ip_ver))
 load(pathJoin("sigio", sigio_ver))
 load(pathJoin("sfcio", sfcio_ver))
 load(pathJoin("nemsio", nemsio_ver))
-load(pathJoin("wrf_io", wrf_io_ver))
+load(pathJoin("wrf-io", wrf_io_ver))
 load(pathJoin("ncio", ncio_ver))
 load(pathJoin("crtm", crtm_ver))
-load(pathJoin("ncdiag",ncdiag_ver))
+load(pathJoin("gsi-ncdiag",ncdiag_ver))
diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua
@@ -1,20 +1,28 @@
 help([[
 ]])
 
-load("cmake/3.20.1")
+unload("intel")
+unload("cray-mpich")
+unload("cray-python")
+unload("darshan")
 
-prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack")
-load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles")
 
-load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2"))
-load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20"))
-load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2"))
-load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20"))
+local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
+local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "7.7.20"
+local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+load(pathJoin("stack-python", stack_python_ver))
+load(pathJoin("cmake", cmake_ver))
 
 load("gsi_common")
 
 local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
-load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("prod-util", prod_util_ver))
 
 -- Needed at runtime:
 load("alps")