Merge pull request #84 from JeffBeck-NOAA/community_develop

Commit to add Hera support
ufs-community · Sep 5, 2019 · be58cec · be58cec
2 parents 22ceff7 + c2884ff
commit be58cec
Show file tree

Hide file tree

Showing 10 changed files with 154 additions and 4 deletions.
diff --git a/jobs/JREGIONAL_MAKE_IC_LOWBC b/jobs/JREGIONAL_MAKE_IC_LOWBC
@@ -141,6 +141,39 @@ case "$MACHINE" in
   { restore_shell_opts; } > /dev/null 2>&1
   ;;
 #
+"HERA")
+#
+  { save_shell_opts; set +x; } > /dev/null 2>&1
+
+  ulimit -s unlimited
+#  ulimit -a
+
+  module purge
+  module load intel/18.0.5.274
+  module load impi/2018.0.4
+  module load netcdf/4.6.1
+
+  module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
+  module load esmf/8.0.0bs21
+
+  module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
+  module load w3nco
+  module load nemsio
+  module load bacio
+  module load sp
+  module load sfcio
+  module load sigio
+
+  module contrib wrap-mpi
+
+  module list
+
+  np=${SLURM_NTASKS}
+  APRUN="mpirun -np ${np}"
+
+  { restore_shell_opts; } > /dev/null 2>&1
+  ;;
+#
 "JET")
 #
   { save_shell_opts; set +x; } > /dev/null 2>&1

diff --git a/jobs/JREGIONAL_MAKE_LBC1_TO_LBCN b/jobs/JREGIONAL_MAKE_LBC1_TO_LBCN
@@ -132,6 +132,40 @@ case "$MACHINE" in
   { restore_shell_opts; } > /dev/null 2>&1
   ;;
 #
+"HERA")
+#
+  { save_shell_opts; set +x; } > /dev/null 2>&1
+
+  ulimit -s unlimited
+#  ulimit -a
+
+  module purge
+  module load intel/18.0.5.274
+  module load impi/2018.0.4
+  module load netcdf/4.6.1
+
+  module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
+  module load esmf/8.0.0bs21
+
+  module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
+  module load w3nco
+  module load nemsio
+  module load bacio
+  module load sp
+  module load sfcio
+  module load sigio
+
+  module contrib wrap-mpi
+
+  module list
+
+  np=${SLURM_NTASKS}
+  APRUN="mpirun -np ${np}"
+
+  { restore_shell_opts; } > /dev/null 2>&1
+  ;;
+
+#
 "JET")
 #
   { save_shell_opts; set +x; } > /dev/null 2>&1

diff --git a/modulefiles/regional_workflow/orog.hera b/modulefiles/regional_workflow/orog.hera
@@ -4,7 +4,6 @@
 # Loading Intel Compiler Suite
 module purge
 module load intel/18.0.5.274
-
 module load netcdf/4.6.1
 
 # Loading nceplibs modules

diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh
@@ -152,6 +152,18 @@ module list
   APRUN_SFC="mpirun -np ${SLURM_NTASKS}"
   ;;
 
+"HERA")
+module purge
+module load intel/18.0.5.274
+module load impi/2018.0.4
+module load netcdf/4.6.1
+module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
+module load esmf/7.1.0r
+module contrib wrap-mpi
+module list
+  APRUN_SFC="mpirun -np ${SLURM_NTASKS}"
+  ;;
+
 *)
   print_err_msg_exit "${script_name}" "\
 Run command has not been specified for this machine:

diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh
@@ -18,7 +18,7 @@ RUN_ENVIR="nco"
 #
 # MACHINE:
 # Machine on which the workflow will run.  Valid values are "WCOSS_C", 
-# "WCOSS", "DELL", "THEIA", "JET", "ODIN", and "CHEYENNE".  New values 
+# "WCOSS", "DELL", "THEIA","HERA","JET", "ODIN", and "CHEYENNE".  New values 
 # may be added as the workflow is ported to additional machines.
 #
 # ACCOUNT:

diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh
@@ -495,6 +495,9 @@ bination of external model (extrn_mdl_name) and analysis or forecast
     "THEIA")
       sysdir=""
       ;;
+    "HERA")
+      sysdir=""
+      ;;
     "JET")
       sysdir=""
       ;;
@@ -521,6 +524,9 @@ has not been specified for this external model and machine combination:
     "THEIA")
       sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}"
       ;;
+    "HERA")
+      sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}"
+      ;;
     "JET")
       sysdir="$sysbasedir/${yyyymmdd}"
       ;;
@@ -547,6 +553,9 @@ has not been specified for this external model and machine combination:
     "THEIA")
       sysdir="$sysbasedir"
       ;;
+    "HERA")
+      sysdir="$sysbasedir"
+      ;;
     "JET")
       sysdir="$sysbasedir"
       ;;
@@ -573,6 +582,9 @@ has not been specified for this external model and machine combination:
     "THEIA")
       sysdir="$sysbasedir"
       ;;
+    "HERA")
+      sysdir="$sysbasedir"
+      ;;
     "JET")
       sysdir="$sysbasedir"
       ;;

diff --git a/ush/make_grid_orog.sh b/ush/make_grid_orog.sh
@@ -226,6 +226,26 @@ case $MACHINE in
   ulimit -a
   ;;
 
+"HERA")
+#
+  { save_shell_opts; set +x; } > /dev/null 2>&1
+
+  . /apps/lmod/lmod/init/sh
+  module purge
+  module load intel/18.0.5.274
+  module load netcdf/4.6.1
+  module load hdf5/1.10.4
+  module list
+
+  { restore_shell_opts; } > /dev/null 2>&1
+
+  export APRUN="time"
+  export topo_dir="/scratch1/NCEPDEV/global/glopara/fix/fix_orog"
+
+  ulimit -s unlimited
+  ulimit -a
+  ;;
+
 
 "JET")
 #
@@ -562,7 +582,7 @@ $TMPDIR" \
   ;;
 
 
-"THEIA" | "JET" | "ODIN")
+"THEIA" | "HERA" | "JET" | "ODIN")
 # NOTE:  We undefined TMPDIR, but things still seem to work.  WHY???
   $USHDIR/$orog_gen_scr \
     $RES $tile $WORKDIR_GRID $WORKDIR_OROG $USHDIR $topo_dir $TMPDIR || \

diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh
@@ -76,6 +76,9 @@ else
     "THEIA")
       EXTRN_MDL_FILES_SYSBASEDIR_ICSSURF=""
       ;;
+    "HERA")
+      EXTRN_MDL_FILES_SYSBASEDIR_ICSSURF=""
+      ;;
     "JET")
       EXTRN_MDL_FILES_SYSBASEDIR_ICSSURF=""
       ;;
@@ -103,6 +106,9 @@ for this machine and external model combination:
     "THEIA")
       EXTRN_MDL_FILES_SYSBASEDIR_ICSSURF="/scratch4/NCEPDEV/rstprod/com/gfs/prod"
       ;;
+    "HERA")
+      EXTRN_MDL_FILES_SYSBASEDIR_ICSSURF="/scratch1/NCEPDEV/rstprod/com/gfs/prod"
+      ;;
     "JET")
       EXTRN_MDL_FILES_SYSBASEDIR_ICSSURF="/lfs3/projects/hpc-wof1/ywang/regional_fv3/gfs"
       ;;
@@ -127,6 +133,9 @@ for this machine and external model combination:
     "THEIA")
       EXTRN_MDL_FILES_SYSBASEDIR_ICSSURF="/scratch4/BMC/public/data/gsd/rr/full/wrfnat"
       ;;
+    "HERA")
+      EXTRN_MDL_FILES_SYSBASEDIR_ICSSURF="/scratch2/BMC/public/data/gsd/rap/full/wrfnat"
+      ;;
     *)
       print_err_msg_exit "\
 The system directory in which to look for the files generated by the ex-
@@ -145,6 +154,9 @@ for this machine and external model combination:
     "THEIA")
       EXTRN_MDL_FILES_SYSBASEDIR_ICSSURF="/scratch4/BMC/public/data/gsd/hrrr/conus/wrfnat"
       ;;
+    "HERA")
+      EXTRN_MDL_FILES_SYSBASEDIR_ICSSURF="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat"
+      ;;
     *)
       print_err_msg_exit "\
 The system directory in which to look for the files generated by the ex-
@@ -229,6 +241,9 @@ else
     "THEIA")
       EXTRN_MDL_FILES_SYSBASEDIR_LBCS=""
       ;;
+    "HERA")
+      EXTRN_MDL_FILES_SYSBASEDIR_LBCS=""
+      ;;
     "JET")
       EXTRN_MDL_FILES_SYSBASEDIR_LBCS=""
       ;;
@@ -257,6 +272,9 @@ else
     "THEIA")
       EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch4/NCEPDEV/rstprod/com/gfs/prod"
       ;;
+    "HERA")
+      EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch1/NCEPDEV/rstprod/com/gfs/prod"
+      ;;
     "JET")
       EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/lfs3/projects/hpc-wof1/ywang/regional_fv3/gfs"
       ;;
@@ -282,6 +300,9 @@ else
     "THEIA")
       EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch4/BMC/public/data/gsd/rr/full/wrfnat"
       ;;
+    "HERA")
+      EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat"
+      ;;
     *)
       print_err_msg_exit "\
   The system directory in which to look for the files generated by the ex-
@@ -302,6 +323,11 @@ else
     "THEIA")
       EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch4/BMC/public/data/gsd/hrrr/conus/wrfnat"
       ;;
+    "HERA")
+      EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat"
+      ;;
+
+
     *)
       print_err_msg_exit "\
   The system directory in which to look for the files generated by the ex-

diff --git a/ush/setup.sh b/ush/setup.sh
@@ -306,6 +306,15 @@ Then remove this message and rerun."
   QUEUE_RUN_FV3SAR=${QUEUE_RUN_FV3SAR:-""}
   ;;
 #
+"HERA")
+#
+  ncores_per_node=24
+  SCHED="slurm"
+  QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"}
+  QUEUE_HPSS=${QUEUE_HPSS:-"service"}
+  QUEUE_RUN_FV3SAR=${QUEUE_RUN_FV3SAR:-""}
+  ;;
+#
 "JET")
 #
   ncores_per_node=24
@@ -645,6 +654,11 @@ case $MACHINE in
 #  fi
   ;;
 
+"HERA")
+  FIXgsm="/scratch1/NCEPDEV/global/glopara/fix/fix_am"
+  SFC_CLIMO_INPUT_DIR="/scratch1/NCEPDEV/da/George.Gayno/climo_fields_netcdf"
+  ;;
+
 "JET")
   FIXgsm="/lfs3/projects/hpc-wof1/ywang/regional_fv3/fix/fix_am"
   ;;

diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh
@@ -1,6 +1,6 @@
 valid_vals_RUN_ENVIR=("nco" "community")
 valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")
-valid_vals_MACHINE=("WCOSS_C" "WCOSS" "DELL" "THEIA" "JET" "ODIN" "CHEYENNE")
+valid_vals_MACHINE=("WCOSS_C" "WCOSS" "DELL" "THEIA" "HERA" "JET" "ODIN" "CHEYENNE")
 valid_vals_predef_domain=("GSD_HAFSV0.A" "GSD_RAP13km" "GSD_HRRR25km" "GSD_HRRR13km" "GSD_HRRR3km" "EMC_CONUS" "EMC_AK")
 valid_vals_CCPP=("true" "false")
 valid_vals_CCPP_phys_suite=("GFS" "GSD")