Revert "Revert "Added support on Odin and Stampede (#242)" (#243)" (#244

) This reverts commit ad4d76e.
ufs-community · Jun 26, 2020 · cc62987 · cc62987
1 parent ad4d76e
commit cc62987
Show file tree

Hide file tree

Showing 23 changed files with 200 additions and 6 deletions.
diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS
@@ -85,12 +85,16 @@ case "$MACHINE" in
     ;;
 
   "ODIN")
+    APRUN="srun"
     ;;
 
   "CHEYENNE")
     nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS ))
     APRUN="mpirun -np $nprocs"
     ;;
+  "STAMPEDE")
+    APRUN="ibrun"
+    ;;
 
 esac
 #

diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS
@@ -85,13 +85,18 @@ case "$MACHINE" in
     ;;
 
   "ODIN")
+    APRUN="srun"
     ;;
 
   "CHEYENNE")
     nprocs=$(( NNODES_MAKE_LBCS*PPN_MAKE_LBCS ))
     APRUN="mpirun -np $nprocs"
     ;;
 
+  "STAMPEDE")
+    APRUN="ibrun"
+    ;;
+
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/modulefiles/codes/odin/global_equiv_resol b/modulefiles/codes/odin/global_equiv_resol
@@ -0,0 +1,13 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module load PrgEnv-intel
+module swap intel/19.0.5.281
+module load cray-mpich/7.7.10
+module load cray-libsci
+module load cray-netcdf-hdf5parallel
+module load cray-parallel-netcdf
+module load cray-hdf5-parallel
+
+export NETCDF=/opt/cray/pe/netcdf-hdf5parallel/4.6.3.2/INTEL/19.0
+export HDF5=/opt/cray/pe/hdf5-parallel/1.10.5.2/INTEL/19.0
diff --git a/modulefiles/codes/odin/mosaic_file b/modulefiles/codes/odin/mosaic_file
@@ -0,0 +1,13 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module load PrgEnv-intel
+module swap intel/19.0.5.281
+module load cray-mpich/7.7.10
+module load cray-libsci
+module load cray-netcdf-hdf5parallel
+module load cray-parallel-netcdf
+module load cray-hdf5-parallel
+
+export NETCDF=/opt/cray/pe/netcdf-hdf5parallel/4.6.3.2/INTEL/19.0
+export HDF5=/opt/cray/pe/hdf5-parallel/1.10.5.2/INTEL/19.0
diff --git a/modulefiles/codes/odin/regional_grid b/modulefiles/codes/odin/regional_grid
@@ -0,0 +1,11 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module load PrgEnv-intel
+module swap intel/19.0.5.281
+module load cray-mpich/7.7.10
+module load cray-libsci
+module load cray-netcdf-hdf5parallel
+module load cray-parallel-netcdf
+module load cray-hdf5-parallel
+
diff --git a/modulefiles/codes/stampede/global_equiv_resol b/modulefiles/codes/stampede/global_equiv_resol
@@ -0,0 +1,10 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module load intel/18.0.2
+module load impi/18.0.2
+module load hdf5/1.10.4
+module load netcdf/4.6.2
+
+setenv NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64
+setenv HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64
diff --git a/modulefiles/codes/stampede/mosaic_file b/modulefiles/codes/stampede/mosaic_file
@@ -0,0 +1,12 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+
+
+module load intel/18.0.2
+module load impi/18.0.2
+module load hdf5/1.10.4
+module load netcdf/4.6.2
+
+setenv  NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64
+setenv  HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64
diff --git a/modulefiles/codes/stampede/regional_grid b/modulefiles/codes/stampede/regional_grid
@@ -0,0 +1,10 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module load intel/18.0.2
+module load impi/18.0.2
+module load hdf5/1.10.4
+module load netcdf/4.6.2
+
+setenv NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64
+setenv HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64
diff --git a/modulefiles/tasks/odin/get_extrn_ics b/modulefiles/tasks/odin/get_extrn_ics
@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for get_extrn_ics task.
+#############################################################
+
+#module purge
+
+#module load hpss
+
diff --git a/modulefiles/tasks/odin/get_extrn_lbcs b/modulefiles/tasks/odin/get_extrn_lbcs
@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for get_extrn_lbcs task.
+#############################################################
+
+#module purge
+
+#module load hpss
+
diff --git a/modulefiles/tasks/odin/make_grid b/modulefiles/tasks/odin/make_grid
@@ -0,0 +1,14 @@
+#%Module#####################################################
+## Module file for make_grid task.
+#############################################################
+
+#module purge
+
+module load PrgEnv-intel
+module swap intel/19.0.5.281
+module load cray-libsci
+module load cray-netcdf-hdf5parallel
+module load cray-parallel-netcdf
+module load cray-hdf5-parallel
+module load gcc/6.1.0
+module load slurm
diff --git a/modulefiles/tasks/stampede/get_extrn_ics b/modulefiles/tasks/stampede/get_extrn_ics
@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for get_extrn_ics task.
+#############################################################
+
+#module purge
+
+#module load hpss
+
diff --git a/modulefiles/tasks/stampede/get_extrn_lbcs b/modulefiles/tasks/stampede/get_extrn_lbcs
@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for get_extrn_lbcs task.
+#############################################################
+
+#module purge
+
+#module load hpss
+
diff --git a/modulefiles/tasks/stampede/make_grid b/modulefiles/tasks/stampede/make_grid
@@ -0,0 +1,11 @@
+#%Module#####################################################
+## Module file for make_grid task.
+#############################################################
+
+module purge
+
+module load intel/18.0.2
+module load impi/18.0.2
+module load hdf5/1.10.4
+module load netcdf/4.6.2
+module load python3/3.7.0
diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh
@@ -148,6 +148,13 @@ case $MACHINE in
   APRUN="time"
   ;;
 
+"STAMPEDE")
+#
+  export APRUN="time"
+
+  ulimit -s unlimited
+  ulimit -a
+  ;;
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh
@@ -145,6 +145,11 @@ case $MACHINE in
   APRUN="time"
   ;;
 
+"STAMPEDE")
+  export APRUN="time"
+  export topo_dir="/work/00315/tg455890/stampede2/regional_fv3/fix_orog"
+  ;;
+
 esac
 #
 #-----------------------------------------------------------------------
@@ -317,7 +322,7 @@ ${tmp_dir}" \
   ;;
 
 
-"CHEYENNE" | "HERA" | "JET" | "ODIN")
+"CHEYENNE" | "HERA" | "JET" | "ODIN" | "STAMPEDE")
   $APRUN "${exec_fp}" < "${input_redirect_fn}" || \
     print_err_msg_exit "\
 Call to executable (exec_fp) that generates the raw orography file returned 

diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh
@@ -151,6 +151,16 @@ case $MACHINE in
   APRUN="mpirun -np $nprocs"
   ;;
 
+"ODIN")
+  nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO ))
+  APRUN="srun -n $nprocs"
+  ;;
+
+"STAMPEDE")
+  nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO ))
+  APRUN="ibrun -np ${nprocs}"
+  ;;
+
 *)
   print_err_msg_exit "\
 Run command has not been specified for this machine:

diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh
@@ -120,6 +120,15 @@ case $MACHINE in
   LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
   ;;
 #
+
+"STAMPEDE")
+#
+  module list
+
+  APRUN="ibrun -np ${PE_MEMBER01}"
+  #LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+  ;;
+
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh
@@ -128,6 +128,11 @@ case $MACHINE in
   APRUN="mpirun -np $nprocs"
   ;;
 
+"STAMPEDE")
+  nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST ))
+  APRUN="ibrun -n $nprocs"
+  ;;
+
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh
@@ -545,11 +545,14 @@ bination of external model (extrn_mdl_name) and analysis or forecast
       sysdir=""
       ;;
     "ODIN")
-      sysdir=""
+      sysdir="$sysbasedir"
       ;;
     "CHEYENNE")
       sysdir=""
       ;;
+    "STAMPEDE")
+      sysdir="$sysbasedir"
+      ;;
     *)
       print_err_msg_exit "\
 The system directory in which to look for external model output files 

diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh
@@ -68,11 +68,14 @@ else
       EXTRN_MDL_FILES_SYSBASEDIR_ICS=""
       ;;
     "ODIN")
-      EXTRN_MDL_FILES_SYSBASEDIR_ICS=""
+      EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/ywang/EPIC/GDAS/2019053000_mem001"
       ;;
     "CHEYENNE")
       EXTRN_MDL_FILES_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS"
       ;;
+    "STAMPEDE")
+      EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001"
+      ;;
     *)
       print_err_msg_exit "\
 The system directory in which to look for the files generated by the ex-
@@ -223,11 +226,14 @@ else
       EXTRN_MDL_FILES_SYSBASEDIR_LBCS=""
       ;;
     "ODIN")
-      EXTRN_MDL_FILES_SYSBASEDIR_LBCS=""
+      EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/ywang/EPIC/GDAS/2019053000_mem001"
       ;;
     "CHEYENNE")
       EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS"
       ;;
+    "STAMPEDE")
+      EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001"
+      ;;
     *)
       print_err_msg_exit "\
 The system directory in which to look for the files generated by the ex-

diff --git a/ush/setup.sh b/ush/setup.sh
@@ -312,7 +312,16 @@ case $MACHINE in
   QUEUE_HPSS=${QUEUE_HPSS:-"regular"}
   QUEUE_HPSS_TAG="queue"       # pbspro does not support "partition" tag
   QUEUE_FCST=${QUEUE_FCST:-"regular"}
+  ;;
+#
+"STAMPEDE")
 #
+  NCORES_PER_NODE=68
+  SCHED="slurm"
+  QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"}
+  QUEUE_HPSS=${QUEUE_HPSS:-"development"}
+  QUEUE_FCST=${QUEUE_FCST:-"normal"}
+  ;;
 esac
 #
 #-----------------------------------------------------------------------
@@ -670,13 +679,18 @@ case $MACHINE in
 
 "ODIN")
   FIXgsm="/scratch/ywang/fix/theia_fix/fix_am"
-  SFC_CLIMO_INPUT_DIR="/scratch1/NCEPDEV/da/George.Gayno/ufs_utils.git/climo_fields_netcdf"
+  SFC_CLIMO_INPUT_DIR="/scratch/ywang/fix/climo_fields_netcdf"
   ;;
 "CHEYENNE")
   FIXgsm="/glade/p/ral/jntp/UFS_CAM/fix/fix_am"
   SFC_CLIMO_INPUT_DIR="/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf"
   ;;
 
+"STAMPEDE")
+  FIXgsm="/work/00315/tg455890/stampede2/regional_fv3/fix_am"
+  SFC_CLIMO_INPUT_DIR="/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"
+  ;;
+
 *)
   print_err_msg_exit "\
 Directories have not been specified for this machine:

diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh
@@ -1,6 +1,6 @@
 valid_vals_RUN_ENVIR=("nco" "community")
 valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")
-valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "THEIA" "HERA" "JET" "ODIN" "CHEYENNE")
+valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "THEIA" "HERA" "JET" "ODIN" "CHEYENNE" "STAMPEDE")
 valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none")
 valid_vals_PREDEF_GRID_NAME=( \
 "EMC_CONUS_3km" \