Merge pull request #316 from NOAA-EMC/feature/hpc-stack

GFS components update for hpc-stack support
NOAA-EMC · Jun 23, 2021 · 96347ea · 96347ea
2 parents cfca8bb + 5c042e0
commit 96347ea
Show file tree

Hide file tree

Showing 30 changed files with 410 additions and 176 deletions.
diff --git a/.gitignore b/.gitignore
@@ -33,6 +33,7 @@ sorc/logs
 sorc/fv3gfs.fd
 sorc/gfs_post.fd
 sorc/gfs_wafs.fd
+sorc/gldas.fd
 sorc/gsi.fd
 sorc/ufs_utils.fd
 sorc/verif-global.fd
@@ -44,14 +45,12 @@ sorc/calc_increment_ens.fd
 sorc/calc_increment_ens_ncio.fd
 sorc/emcsfc_ice_blend.fd
 sorc/emcsfc_snow2mdl.fd
-sorc/filter_topo.fd
 sorc/fregrid.fd
 sorc/gdas2gldas.fd
 sorc/getsfcensmeanp.fd
 sorc/getsigensmeanp_smooth.fd
 sorc/getsigensstatp.fd
 sorc/gfs_ncep_post.fd
-sorc/gldas.fd
 sorc/gldas2gdas.fd
 sorc/gldas_forcing.fd
 sorc/gldas_model.fd
@@ -64,21 +63,15 @@ sorc/global_gsi.fd
 sorc/interp_inc.fd
 sorc/make_hgrid.fd
 sorc/make_solo_mosaic.fd
-sorc/mkgfsnemsioctl.fd
 sorc/ncdiag_cat.fd
-sorc/nemsio_chgdate.fd
-sorc/nemsio_get.fd
-sorc/nemsio_read.fd
 sorc/nst_tf_chg.fd
-sorc/orog.fd
 sorc/oznmon_horiz.fd
 sorc/oznmon_time.fd
 sorc/radmon_angle.fd
 sorc/radmon_bcoef.fd
 sorc/radmon_bcor.fd
 sorc/radmon_time.fd
 sorc/recentersigp.fd
-sorc/shave.fd
 sorc/wafs_awc_wafavn.fd
 sorc/wafs_blending.fd
 sorc/wafs_blending_0p25.fd
@@ -144,10 +137,10 @@ scripts/exglobal_diag.sh
 # ush symlinks
 ush/calcanl_gfs.py
 ush/calcinc_gfs.py
+ush/chgres_cube.sh
 ush/emcsfc_ice_blend.sh
 ush/emcsfc_snow.sh
 ush/fix_precip.sh
-ush/fv3gfs_chgres.sh
 ush/fv3gfs_downstream_nems.sh
 ush/fv3gfs_driver_grid.sh
 ush/fv3gfs_dwn_nems.sh

diff --git a/Externals.cfg b/Externals.cfg
@@ -1,35 +1,36 @@
 # External sub-modules of global-workflow
 
 [FV3GFS]
-tag = GFS.v16.0.16
+hash = 9350745855aebe0790813e0ed2ba5ad680e3f75c
 local_path = sorc/fv3gfs.fd
 repo_url = https://github.com/ufs-community/ufs-weather-model.git
 protocol = git
 required = True
 
 [GSI]
-tag = gfsda.v16.1.0
+hash = 9c1fc15d42573b398037319bbf8d5143ad126fb6
 local_path = sorc/gsi.fd
 repo_url = https://github.com/NOAA-EMC/GSI.git
 protocol = git
 required = True
 
 [GLDAS]
-tag = gldas_gfsv16_release.v1.13.0
+tag = gldas_gfsv16_release.v1.15.0
 local_path = sorc/gldas.fd
 repo_url = https://github.com/NOAA-EMC/GLDAS.git
 protocol = git
 required = True
 
 [EMC_post]
-tag = upp_gfsv16_release.v1.1.3
+#No externals setting = .gitmodules will be invoked for CMakeModules and comupp/src/lib/crtm2 submodules
+hash = ff42e0227d6100285d4179a2572b700fd5a959cb
 local_path = sorc/gfs_post.fd
 repo_url = https://github.com/NOAA-EMC/EMC_post.git
 protocol = git
 required = True
 
 [UFS_UTILS]
-tag = ops-gfsv16.0.0
+tag = ufs_utils_1_4_0
 local_path = sorc/ufs_utils.fd
 repo_url = https://github.com/NOAA-EMC/UFS_UTILS.git
 protocol = git
@@ -43,7 +44,7 @@ protocol = git
 required = True
 
 [EMC_gfs_wafs]
-tag = gfs_wafs.v6.0.22
+hash = c2a29a67d9432b4d6fba99eac7797b81d05202b6
 local_path = sorc/gfs_wafs.fd
 repo_url = https://github.com/NOAA-EMC/EMC_gfs_wafs.git
 protocol = git

diff --git a/env/HERA.env b/env/HERA.env
@@ -160,14 +160,6 @@ elif [ $step = "efcs" ]; then
 
 elif [ $step = "post" ]; then
 
-    source /apps/lmod/lmod/init/sh
-    module unload esmf/8.0.1_ParallelNetCDF.release
-    module unload netcdf_parallel/4.7.4.release
-    module unload hdf5_parallel/1.10.6.release
-    module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-    module load hdf5_parallel/1.10.6
-    module load netcdf_parallel/4.7.4
-
     nth_max=$(($npe_node_max / $npe_node_post))
 
     export NTHREADS_NP=${nth_np:-1}

diff --git a/modulefiles/OznMonBuild.orion b/modulefiles/OznMonBuild.orion
@@ -10,8 +10,5 @@ export FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl"
 
 export D_FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl -debug"
 
-module load intel/2018.4
-#module load w3nco/2.0.6
-module use -a /work/noaa/noaatest/NCEPLIBS/lib/modulefiles
-module load w3nco-intel-sandybridge/2.0.6
-
+module load hpc-intel/2018.4
+module load w3nco/2.4.1
diff --git a/modulefiles/OznMonBuild.wcoss_dell_p3 b/modulefiles/OznMonBuild.wcoss_dell_p3
@@ -10,6 +10,8 @@ export FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl"
 
 export D_FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl -debug"
 
-module load  ips/18.0.1.163
-module load w3nco/2.0.6
-
+module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-ips/18.0.1.163
+module load hpc-impi/18.0.1
+module load w3nco/2.4.1
diff --git a/modulefiles/RadMonBuild.orion b/modulefiles/RadMonBuild.orion
@@ -11,7 +11,5 @@ export FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -as
 export CHECK_LIBS="/gpfs/hps/nco/ops/nwprod/spa_util/check_libs.bash"
 export D_FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -assume byterecl -debug"
 
-module load intel/2018.4
-#module load w3nco/2.0.6
-module use -a /work/noaa/noaatest/NCEPLIBS/lib/modulefiles
-module load w3nco-intel-sandybridge/2.0.6
+module load hpc-intel/2018.4
+module load w3nco/2.4.1
diff --git a/modulefiles/RadMonBuild.wcoss_dell_p3 b/modulefiles/RadMonBuild.wcoss_dell_p3
@@ -11,5 +11,8 @@ export FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -as
 export CHECK_LIBS="/gpfs/hps/nco/ops/nwprod/spa_util/check_libs.bash"
 export D_FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -assume byterecl -debug"
 
-module load ips/18.0.1.163
-module load w3nco/2.0.6
+module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-ips/18.0.1.163
+module load hpc-impi/18.0.1
+module load w3nco/2.4.1
diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
@@ -3,28 +3,49 @@
 ##      FV3GFS prerequisites
 ##
 
-module load intel/18.0.5.274
-module load impi/2018.0.4
-module load wgrib2/2.0.8
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
+
 module load hpss/hpss
-module load nco/4.7.0
+module load nco/4.9.1
 module load gempak/7.4.2
 
-#Load from official NCEPLIBS
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load prod_util/1.1.0
-module load grib_util/1.1.1
-module load g2tmpl/1.6.0
+#Load from hpc-stack
+module load prod_util/1.2.2
+module load grib_util/1.2.2
+
 module load crtm/2.3.0
+setenv CRTM_FIX /scratch2/NCEPDEV/nwprod/NCEPLIBS/fix/crtm_v2.3.0
+
+module load jasper/2.0.25
+module load zlib/1.2.11
+module load png/1.6.35
 
-module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles/
-module load netcdf_parallel/4.7.4.release
-module load hdf5_parallel/1.10.6.release
-module load esmf/8.0.1_ParallelNetCDF.release
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load pio/2.5.2
+module load esmf/8_1_1
+module load fms/2020.04.03
+
+module load bacio/2.4.1
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.6
+
+module load wgrib2/2.0.8
+setenv WGRIB2 wgrib2
 
 # python
 module use -a /contrib/anaconda/modulefiles
 module load anaconda/2.3.0
 
 # waveprep
-module load cdo/1.9.5
+module load cdo/1.9.5 
+
diff --git a/modulefiles/module_base.orion b/modulefiles/module_base.orion
@@ -3,37 +3,50 @@
 ##      FV3GFS prerequisites
 ##
 
-module load intel/2018.4
-module load impi/2018.4
-module load wgrib/2.0.8
+module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/2018.4
+module load hpc-impi/2018.4
+
 #module load hpss/hpss
 module load nco/4.8.1
-module load cdo/1.9.5
-
 module load gempak/7.5.1
 
-#Load from official NCEPLIBS
-module use /apps/contrib/NCEPLIBS/orion/modulefiles
-module load grib_util/1.2.0
-module load prod_util/1.2.0
-module load g2tmpl/1.6.0
+module load grib_util/1.2.2
+module load prod_util/1.2.2
+
 module load crtm/2.3.0
-#module load netcdf_parallel/4.7.4
-module load hdf5_parallel/1.10.6
-
-## load ESMF library for above compiler / MPI combination
-### use pre-compiled EMSF library for above compiler / MPI combination
-###
-module use /apps/contrib/NCEPLIBS/lib/modulefiles
-module load netcdfp/4.7.4 
-#module load hdf5_parallel/1.10.6
-module load esmflocal/8.0.1.08bs
-module load post-intel-sandybridge/8.0.5
+setenv CRTM_FIX /apps/contrib/NCEPLIBS/orion/fix/crtm_v2.3.0
+
+module load jasper/2.0.25
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load pio/2.5.2
+module load esmf/8_1_1
+module load fms/2020.04.03
+
+module load bacio/2.4.1
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.6
+
+module load wgrib/2.0.8
+setenv WGRIB2 wgrib2
 
 module load contrib
-module load rocoto/1.3.2
+module load rocoto/1.3.3
 module load slurm/19.05.3-2
 
 # Python
 module load python/3.7.5
 
+# waveprep
+module load cdo/1.9.5
diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
@@ -4,30 +4,53 @@
 
 # From default environment
 
-module load ips/18.0.1.163
-module load impi/18.0.1    
+module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-ips/18.0.1.163
+module load hpc-impi/18.0.1
+
 module load lsf/10.1       
 module load EnvVars/1.0.3
-
 module load HPSS/5.0.2.5
-module load prod_util/1.1.4
+
+module load prod_util/1.2.2
 module load prod_envir/1.1.0
-module unload grib_util/1.0.6   
-module load grib_util/1.1.0
+module load grib_util/1.2.2
+module load util_shared/1.3.0
+
+module load crtm/2.3.0
+setenv CRTM_FIX /gpfs/dell1/nco/ops/nwprod/lib/crtm/v2.3.0/fix
 
 module load NCO/4.7.0   
 module load CFP/2.0.2
 setenv USE_CFP YES
 module load pm5
 
-module load NetCDF-parallel/4.7.4
-module load ESMF/8.0.1
-module load HDF5-parallel/1.10.6
+module load jasper/2.0.25
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load pio/2.5.2
+module load esmf/8_1_1
+module load fms/2020.04.03
+
+module load bacio/2.4.1
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.6
+
+module load wgrib2/2.0.8
+setenv WGRIB2 wgrib2
 
 module use -a /gpfs/dell1/nco/ops/nwprod/modulefiles/
 module load gempak/7.3.3
-module load g2tmpl/1.6.0
-module load util_shared/1.3.0
 
 # Load for WAFS
 module load bufr_dumplist/2.0.0
@@ -36,8 +59,5 @@ module load dumpjb/5.1.0
 # python
 module load python/3.6.3
 
-# Load crtm
-module load crtm/2.3.0
-
 # waveprep
 module load cdo/1.9.8