Merge remote-tracking branch 'origin/testing/pdlib-restarts' into tes…

…ting/pdlib-restarts * origin/testing/pdlib-restarts: Fix compiler remarks for ST6 and GMD (NOAA-EMC#1206) For NCEP regtests, add option for gnu compiler and new machine Hercules (NOAA-EMC#1145)
MatthewMasarik-NOAA · Apr 4, 2024 · e1e76c3 · e1e76c3
2 parents 52a9104 + eac9eab
commit e1e76c3
Show file tree

Hide file tree

Showing 3 changed files with 103 additions and 41 deletions.
diff --git a/model/src/w3gridmd.F90 b/model/src/w3gridmd.F90
@@ -6330,14 +6330,14 @@ SUBROUTINE W3GRID()
 2923 FORMAT ( '             ',2F8.3,F6.1,2E12.4)
 2922 FORMAT ( '  &SNL3 NQDEF =',I3,', MSC =',F6.2,',  NSC =',   &
          F6.2,',  KDFD =',F6.2,',  KDFS =',F6.2,' /')
-3923 FORMAT ( '  &ANL3 QPARMS = ',2(F5.3,', '),F5.1,', ',E10.4, &
-         ', ',E10.4,' /')
-4923 FORMAT ( '  &ANL3 QPARMS = ',2(F5.3,', '),F5.1,', ',E10.4, &
-         ', ',E10.4,' ,')
-5923 FORMAT ( '                 ',2(F5.3,', '),F5.1,', ',E10.4, &
-         ', ',E10.4,' ,')
-6923 FORMAT ( '                 ',2(F5.3,', '),F5.1,', ',E10.4, &
-         ', ',E10.4,' /')
+3923 FORMAT ( '  &ANL3 QPARMS = ',2(F5.3,', '),F5.1,', ',E11.4, &
+         ', ',E11.4,' /')
+4923 FORMAT ( '  &ANL3 QPARMS = ',2(F5.3,', '),F5.1,', ',E11.4, &
+         ', ',E11.4,' ,')
+5923 FORMAT ( '                 ',2(F5.3,', '),F5.1,', ',E11.4, &
+         ', ',E11.4,' ,')
+6923 FORMAT ( '                 ',2(F5.3,', '),F5.1,', ',E11.4, &
+         ', ',E11.4,' /')
 #endif
     !
 #ifdef W3_NL4
@@ -6453,18 +6453,18 @@ SUBROUTINE W3GRID()
 925 FORMAT ( '  normalise by threshold spectral density    :  ',A/&
          '  normalise by spectral density              :  ',A/&
          '  coefficient and exponent  for                 '/  &
-         '   inherent breaking term a1, L as in (21)   : ',E9.3,I3/ &
-         '   cumulative breaking term a2, M as in (22) : ',E9.3,I3/ &
+         '   inherent breaking term a1, L as in (21)   : ',E10.3,I3/ &
+         '   cumulative breaking term a2, M as in (22) : ',E10.3,I3/ &
          ' ')
-2924 FORMAT ( '  &SDS6 SDSET = ',L,', SDSA1 = ',E9.3,              &
-         ', SDSA2 = ',E9.3,', SDSP1 = ',I2,', SDSP1 = ',      &
+2924 FORMAT ( '  &SDS6 SDSET = ',L,', SDSA1 = ',E10.3,              &
+         ', SDSA2 = ',E10.3,', SDSP1 = ',I2,', SDSP1 = ',      &
          I2,' /'                                              )
 
 937 FORMAT (/'  Swell dissipation ',A/                            &
          ' --------------------------------------------------')
 940 FORMAT ( '  subroutine W3SWL6 activated           : ',A/      &
-         '   coefficient b1 ',A,                ' : ',E9.3/   )
-2937 FORMAT ( '  &SWL6 SWLB1 = ',E9.3,', CSTB1 = ',L,' /')
+         '   coefficient b1 ',A,                ' : ',E10.3/   )
+2937 FORMAT ( '  &SWL6 SWLB1 = ',E10.3,', CSTB1 = ',L,' /')
 #endif
     !
 #ifdef W3_BT0
@@ -6549,7 +6549,7 @@ SUBROUTINE W3GRID()
 946 FORMAT  ('  Isotropic (linear function of ice concentration)'/&
          '        slope                      : ',E10.3/ &
          '        offset                     : ',E10.3)
-2946 FORMAT ( '  &SIS1 ISC1 =',E9.3,', ISC2 =',E9.3)
+2946 FORMAT ( '  &SIS1 ISC1 =',E10.3,', ISC2 =',E10.3)
 #endif
 #ifdef W3_IS2
 947 FORMAT  (/'  Ice scattering ',A,/ &

diff --git a/model/src/wminitmd.F90 b/model/src/wminitmd.F90
@@ -1273,6 +1273,12 @@ SUBROUTINE WMINIT ( IDSI, IDSO, IDSS, IDST, IDSE, IFNAME,       &
         IF (WORDS(6) .EQ. 'T') THEN
           CALL NEXTLN ( COMSTR , MDSI , MDSE2 )
           READ (MDSI,*,END=2001,ERR=2002)(ODAT(I,1),I=5*(8-1)+1,5*8)
+        ELSE 
+          ODAT(5*(8-1)+1,1)=0
+          ODAT(5*(8-1)+2,1)=0
+          ODAT(5*(8-1)+3,1)=0
+          ODAT(5*(8-1)+4,1)=0
+          ODAT(5*8,1)=0
         END IF
       ELSE
         READ (MDSI,*,END=2001,ERR=2002)(ODAT(I,1),I=5*(J-1)+1,5*J)

diff --git a/regtests/bin/matrix_cmake_ncep b/regtests/bin/matrix_cmake_ncep
@@ -22,59 +22,109 @@ usage ()
 {
   cat 2>&1 << EOF
 
-  Usage: $myname model_dir
+  Usage: $myname model_dir compiler 
   Required:
     model_dir : path to model dir of WW3 source
+  Optional: 
+    compiler : intel (default) or gnu 
 EOF
 }
 
-
 # Get required arguments
   if [ ! $# = 0 ]
   then
     main_dir="$1" ; shift
+    if [ ! $# = 0 ]
+    then
+      compiler="$1"; shift 
+    else 
+      compiler='intel'
+    fi 
   else
     usage
     exit 1
   fi
+
+
 
 # Convert main_dir to absolute path
   main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`"
 
 # Module Versions from spack-stack that are common for all platforms
   modnetcdfc='netcdf-c/4.9.2'
-  modnetcdff='netcdf-fortran/4.6.0'
+  modnetcdff='netcdf-fortran/4.6.1'
   modjasper='jasper/2.0.32'
   modzlib='zlib/1.2.13'
   modpng='libpng/1.6.37'
   modhdf5='hdf5/1.14.0'
   modbacio='bacio/2.4.1'
   modg2='g2/3.4.5'
   modw3emc='w3emc/2.10.0'
-  modesmf='esmf/8.4.2'
+  modesmf='esmf/8.5.0'
   modscotch='scotch/7.0.4'
 
 # Set batchq queue, choose modules and other custom variables to fit system and
 # to define headers etc (default to original version if empty)
   ishera=`hostname | grep hfe`
   isorion=`hostname | grep Orion`
+  ishercules=`hostname | grep hercules`
   if [ $ishera ]
   then
-    # If no other h, assuming Hera
     batchq='slurm'
-    spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core'
-    modcomp='stack-intel/2021.5.0'
-    modmpi='stack-intel-oneapi-mpi/2021.5.1'
-    metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/spack-stack/1.5.0/parmetis-4.0.3/install'
-    modcmake='cmake/3.23.1'
+    if [ $compiler = "intel" ]
+    then 
+      spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core'
+      modcomp='stack-intel/2021.5.0'
+      modmpi='stack-intel-oneapi-mpi/2021.5.1'
+      metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hera/intel/spack-stack/1.6.0/parmetis-4.0.3/install'
+      modcmake='cmake/3.23.1'
+    elif [ $compiler = "gnu" ]
+    then 
+      spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core'
+      modcomp='stack-gcc/9.2.0'
+      modmpi='stack-openmpi/4.1.5'
+      metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hera/gnu/spack-stack/1.6.0/parmetis-4.0.3/install'
+      modcmake='cmake/3.23.1'
+    else 
+      echo "Compiler $compiler not supported on hera" 
+      exit 1 
+    fi 
   elif [ $isorion ]
   then
+    if [ $compiler = "intel" ]
+    then
+      batchq='slurm'
+      spackstackpath='/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core'
+      modcomp='stack-intel/2022.0.2'
+      modmpi='stack-intel-oneapi-mpi/2021.5.1'
+      metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/orion/intel/spack-stack/1.6.0/parmetis-4.0.3/install'
+      modcmake='cmake/3.23.1'
+    else
+      echo "Compiler $compiler not supported on orion" 
+      exit 1
+    fi  
+  elif [ $ishercules ] 
+  then 
     batchq='slurm'
-    spackstackpath='/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core'
-    modcomp='stack-intel/2022.0.2'
-    modmpi='stack-intel-oneapi-mpi/2021.5.1'
-    metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/spack-stack/1.5.0/parmetis-4.0.3/install'
-    modcmake='cmake/3.23.1'
+    if [ $compiler = "intel" ]
+    then
+      spackstackpath='/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core'
+      modcomp='stack-intel/2021.9.0'
+      modmpi='stack-intel-oneapi-mpi/2021.9.0'
+      metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/hercules/intel/spack-stack/1.6.0/parmetis-4.0.3/install'
+      modcmake='cmake/3.23.1'
+    elif [ $compiler = "gnu" ]
+    then 
+      spackstackpath='/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core'
+      spackstackpath2='/work/noaa/epic/role-epic/spack-stack/hercules/modulefiles'
+      modcomp='stack-gcc/12.2.0'
+      modmpi='stack-mvapich2/2.3.7'
+      metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/hercules/gnu/spack-stack/1.6.0/parmetis-4.0.3/install'
+      modcmake='cmake/3.23.1'
+    else
+      echo "Compiler $compiler not supported on hercules" 
+      exit 1
+    fi
   else
     batchq=
   fi
@@ -105,10 +155,19 @@ EOF
     echo '#SBATCH --exclusive'                              >> matrix.head
     echo ' '                                                >> matrix.head
     echo 'ulimit -s unlimited'                              >> matrix.head
-    echo 'ulimit -c 0'                                      >> matrix.head
-    echo 'export KMP_STACKSIZE=2G'                          >> matrix.head
-    echo 'export FI_OFI_RXM_BUFFER_SIZE=128000'             >> matrix.head
-    echo 'export FI_OFI_RXM_RX_SIZE=64000'                  >> matrix.head
+  elif [ $batchq = "slurm" ] && [ $ishercules ]
+  then
+    echo "#SBATCH -n ${np}"                                 >> matrix.head
+    echo "##SBATCH --cpus-per-task=${nth}"                  >> matrix.head
+    echo '#SBATCH -q batch'                                 >> matrix.head
+    echo '#SBATCH -t 08:00:00'                              >> matrix.head
+    echo '#SBATCH -A marine-cpu'                            >> matrix.head
+    echo '#SBATCH -J ww3_regtest'                           >> matrix.head
+    echo '#SBATCH -o matrix.out'                            >> matrix.head
+    echo '#SBATCH -p hercules'                              >> matrix.head
+    echo '#SBATCH --exclusive'                              >> matrix.head
+    echo ' '                                                >> matrix.head
+    echo 'ulimit -s unlimited'                              >> matrix.head
   elif [ $batchq = "slurm" ]
   then
     echo "#SBATCH -n ${np}"                                 >> matrix.head
@@ -133,13 +192,10 @@ EOF
 
 # Netcdf, Parmetis and SCOTCH modules & variables
   echo "  module purge"                                     >> matrix.head
-  if [ ! -z $basemodcomp ]; then
-    echo "  module load $basemodcomp"                       >> matrix.head
-  fi
-  if [ ! -z $basemodmpi ]; then
-    echo "  module load $basemodmpi"                        >> matrix.head
-  fi
-  echo "  module use  $spackstackpath"                      >> matrix.head
+  echo "  module use $spackstackpath"                       >> matrix.head
+  if [ ! -z $spackstackpath2 ]; then
+    echo "  module use $spackstackpath2"                    >> matrix.head
+  fi 
   echo "  module load $modcomp"                             >> matrix.head
   echo "  module load $modmpi"                              >> matrix.head
   echo "  module load $modcmake"                            >> matrix.head