Update RT scripts & modulefiles for removal of Cheyenne

modulefiles/ufs_derecho.intel.lua

@@ -0,0 +1,43 @@
+help([[
+loads UFS Model prerequisites for NOAA Parallelworks/Intel
+]])
+
+
+setenv("LMOD_TMOD_FIND_FIRST","yes")
+prepend_path("MODULEPATH", "/lustre/desc1/scratch/epicufsrt/contrib/modulefiles")
+load("ecflow/5.8.4")
+load("mysql/8.0.33")
+
+setenv("LMOD_TMOD_FIND_FIRST","yes")
+prepend_path("MODULEPATH", "/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra")
+prepend_path("MODULEPATH", "/lustre/desc1/scratch/epicufsrt/contrib/spack-stack/spack-stack-dev-20230825/envs/unified-env/install/modulefiles/Core")
+--prepend_path("MODULEPATH", "/lustre/desc1/scratch/epicufsrt/contrib/spack-stack/spack-stack-dev-20230814/envs/unified-en2/install/modulefiles/Core")
+--prepend_path("MODULEPATH", "/lustre/desc1/scratch/epicufsrt/contrib/spack-stack/spack-stack-dev-20230814/envs/unified-env/install/modulefiles/Core")
+--prepend_path("MODULEPATH", "/lustre/desc1/scratch/mpotts/spack-stack/spack-stack-dev-20230814/envs/ufs-pio-2.5.10/install/modulefiles/Core")
+--prepend_path("MODULEPATH", "/lustre/desc1/scratch/epicufsrt/contrib/spack-stack/spack-stack-1.4.1/envs/unified-dev/install/modulefiles/Core")
+
+--stack_intel_ver=os.getenv("stack_intel_ver") or "2021.8.0"
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+--stack_impi_ver=os.getenv("stack_impi_ver") or "2021.8.0"
+--load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "8.1.25"
+load(pathJoin("stack-cray-mpich", stack_impi_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.26.3"
+load(pathJoin("cmake", cmake_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.10.10"
+load(pathJoin("stack-python", stack_python_ver))
+
+
+setenv("CMAKE_Platform", "derecho.intel")
+load("ufs-weather-model-env")
+--prepend_path("PATH","/opt/cray/pe/pals/1.2.11/bin")
+--setenv("CC", "cc")
+--setenv("CXX", "CC")
+--setenv("FC", "ftn")
+
+whatis("Description: UFS build environment")

tests/compile.sh

@@ -45,7 +45,7 @@ BUILD_DIR=${BUILD_DIR:-$(pwd)/build_${BUILD_NAME}}
 # ----------------------------------------------------------------------
 # Make sure we have reasonable number of threads.
 
-if [[ $MACHINE_ID == cheyenne ]]; then
+if [[ $MACHINE_ID == derecho ]]; then
     BUILD_JOBS=${BUILD_JOBS:-3}
 fi
 

tests/default_vars.sh

@@ -214,62 +214,31 @@ elif [[ $MACHINE_ID = s4 ]]; then
   ICE_tasks_cpl_bmrk=48
   WAV_tasks_cpl_bmrk=80
 
-elif [[ $MACHINE_ID = gaea ]]; then
-
-  TPN=24
+elif [[ $MACHINE_ID = derecho ]]; then
 
+  TPN=128
   INPES_dflt=3 ; JNPES_dflt=8
   INPES_thrd=3 ; JNPES_thrd=4
-  INPES_c384=6 ; JNPES_c384=8  ; THRD_c384=1
+  INPES_c384=8 ; JNPES_c384=6  ; THRD_c384=2
   INPES_c768=8 ; JNPES_c768=16 ; THRD_c768=2
 
-  THRD_cpl_atmw_gdas=3
+  THRD_cpl_atmw_gdas=2
   INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24
-  WAV_tasks_atmw_gdas=264
+  WAV_tasks_atmw_gdas=248
 
-elif [[ $MACHINE_ID = cheyenne ]]; then
+elif [[ $MACHINE_ID = gaea ]]; then
 
-  TPN=36
+  TPN=24
 
   INPES_dflt=3 ; JNPES_dflt=8
   INPES_thrd=3 ; JNPES_thrd=4
-  INPES_c384=8 ; JNPES_c384=6  ; THRD_c384=2
+  INPES_c384=6 ; JNPES_c384=8  ; THRD_c384=1
   INPES_c768=8 ; JNPES_c768=16 ; THRD_c768=2
 
   THRD_cpl_atmw_gdas=3
   INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24
   WAV_tasks_atmw_gdas=264
 
-  THRD_cpl_dflt=1
-  INPES_cpl_dflt=6; JNPES_cpl_dflt=8; WPG_cpl_dflt=6
-  OCN_tasks_cpl_dflt=20
-  ICE_tasks_cpl_dflt=10
-  WAV_tasks_cpl_dflt=20
-
-  THRD_cpl_thrd=2
-  INPES_cpl_thrd=6; JNPES_cpl_thrd=4; WPG_cpl_thrd=6
-  OCN_tasks_cpl_thrd=20
-  ICE_tasks_cpl_thrd=10
-  WAV_tasks_cpl_thrd=20
-
-  THRD_cpl_dcmp=1
-  INPES_cpl_dcmp=4; JNPES_cpl_dcmp=12; WPG_cpl_dcmp=6
-  OCN_tasks_cpl_dcmp=20
-  ICE_tasks_cpl_dcmp=10
-  WAV_tasks_cpl_dcmp=20
-
-  THRD_cpl_mpi=1
-  INPES_cpl_mpi=6; JNPES_cpl_mpi=12; WPG_cpl_mpi=6
-  OCN_tasks_cpl_mpi=24
-  ICE_tasks_cpl_mpi=12
-  WAV_tasks_cpl_mpi=24
-
-  THRD_cpl_bmrk=2
-  INPES_cpl_bmrk=12; JNPES_cpl_bmrk=8; WPG_cpl_bmrk=48
-  OCN_tasks_cpl_bmrk=120
-  ICE_tasks_cpl_bmrk=48
-  WAV_tasks_cpl_bmrk=100
-
 elif [[ $MACHINE_ID = stampede ]]; then
 
   echo "Unknown MACHINE_ID ${MACHINE_ID}. Please update tasks configurations in default_vars.sh"

tests/detect_machine.sh

@@ -75,30 +75,20 @@ case $(hostname -f) in
   Orion-login-3.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion3
   Orion-login-4.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion4
 
+  derecho1.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho1
+  derecho2.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho2
+  derecho3.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho3
+  derecho4.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho4
+  derecho5.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho5
+  derecho6.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho6
+  derecho7.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho7
+  derecho8.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho8
+
   Hercules-login-1.HPC.MsState.Edu) MACHINE_ID=hercules;; ### hercules1 
   Hercules-login-2.HPC.MsState.Edu) MACHINE_ID=hercules;; ### hercules2 
   Hercules-login-3.HPC.MsState.Edu) MACHINE_ID=hercules;; ### hercules3
   Hercules-login-4.HPC.MsState.Edu) MACHINE_ID=hercules;; ### hercules4
 
-  cheyenne1.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1
-  cheyenne2.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne2
-  cheyenne3.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne3
-  cheyenne4.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne4
-  cheyenne5.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne5
-  cheyenne6.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne6
-  cheyenne1.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1
-  cheyenne2.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne2
-  cheyenne3.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne3
-  cheyenne4.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne4
-  cheyenne5.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne5
-  cheyenne6.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne6
-  chadmin1.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1
-  chadmin2.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1
-  chadmin3.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1
-  chadmin4.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1
-  chadmin5.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1
-  chadmin6.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1
-
   login1.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1
   login2.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede2
   login3.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede3

tests/fv3_conf/compile_qsub.IN_derecho

@@ -0,0 +1,16 @@
+#!/bin/bash -l
+#PBS -o out
+#PBS -e err
+#PBS -N @[JBNME]
+#PBS -A @[ACCNR]
+#PBS -q @[QUEUE]
+#PBS -l select=1:ncpus=8:mpiprocs=1
+#PBS -l walltime=00:30:00
+
+set -eux
+
+echo -n " $( date +%s )," >  job_timestamp.txt
+echo "Compile started:  " `date`
+module purge
+
+@[PATHRT]/compile.sh @[MACHINE_ID] "@[MAKE_OPT]" @[COMPILE_NR] @[RT_COMPILER]

tests/fv3_conf/fv3_qsub.IN_cheyenne → tests/fv3_conf/fv3_qsub.IN_derecho

@@ -11,7 +11,7 @@ set -eux
 echo -n " $( date +%s )," >  job_timestamp.txt
 
 set +x
-MACHINE_ID=cheyenne
+MACHINE_ID=derecho
 source ./module-setup.sh
 if [[ ! -d modulefiles ]]; then
   mkdir modulefiles
@@ -30,11 +30,12 @@ export OMP_NUM_THREADS=@[THRD]
 export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
 export ESMF_RUNTIME_PROFILE=ON
 export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
+export MPICH_COLL_OPT_OFF=1
 
 # Avoid job errors because of filesystem synchronization delays
 sync && sleep 1
 
-mpiexec_mpt -p %g: -np @[TASKS] omplace ./fv3.exe
+mpiexec -n @[TASKS] -ppn @[PPN] --hostfile $PBS_NODEFILE ./fv3.exe
 
 echo "Model ended:    " `date`
 echo -n " $( date +%s )," >> job_timestamp.txt