AH MAM5 perlmutter cherry pick E3SM-Project#6

AllenHuAtmoSci · Apr 21, 2023 · 66d8033 · 66d8033
1 parent 9e2474b
commit 66d8033
Show file tree

Hide file tree

Showing 15 changed files with 330 additions and 158 deletions.
diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml
@@ -107,7 +107,7 @@
         </nthrds>
       </pes>
     </mach>
-    <mach name="theta|pm-gpu|cori-knl|cori-haswell|jlse">
+    <mach name="thetapm-gpu|cori-knl|cori-haswell|jlse">
       <pes compset="any" pesize="any">
         <comment>allactive: default, 1 node x MAX_MPITASKS_PER_NODE mpi x 1 omp @ root 0</comment>
         <ntasks>
@@ -122,7 +122,7 @@
         </ntasks>
       </pes>
     </mach>
-    <mach name="pm-cpu|alvarez">
+    <mach name="pm-cpu">
       <pes compset="any" pesize="any">
         <comment>allactive: default, 1 node x MAX_MPITASKS_PER_NODE mpi x 2 omp @ root 0</comment>
         <ntasks>

diff --git a/cime_config/machines/Depends.pm-cpu.gnu.cmake b/cime_config/machines/Depends.pm-cpu.gnu.cmake
@@ -0,0 +1,14 @@
+list(APPEND NO_INLINE_ARG_PACKING_LIST
+  eam/src/dynamics/se/inidat.F90
+)
+
+if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10)
+
+  if (NOT DEBUG)
+    # new in gnu10, inline arg packing was causing INF values with SMS_P4x1.ne4pg2_ne4pg2.F-MMFXX
+    foreach(ITEM IN LISTS NO_INLINE_ARG_PACKING_LIST)
+      e3sm_add_flags("${ITEM}" " -fno-inline-arg-packing")
+    endforeach()
+  endif()
+
+endif()
diff --git a/cime_config/machines/Depends.pm-gpu.gnu.cmake b/cime_config/machines/Depends.pm-gpu.gnu.cmake
@@ -0,0 +1,14 @@
+list(APPEND NO_INLINE_ARG_PACKING_LIST
+  eam/src/dynamics/se/inidat.F90
+)
+
+if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10)
+
+  if (NOT DEBUG)
+    # new in gnu10, inline arg packing was causing INF values with SMS_P4x1.ne4pg2_ne4pg2.F-MMFXX
+    foreach(ITEM IN LISTS NO_INLINE_ARG_PACKING_LIST)
+      e3sm_add_flags("${ITEM}" " -fno-inline-arg-packing")
+    endforeach()
+  endif()
+
+endif()
diff --git a/cime_config/machines/Depends.pm-gpu.gnugpu.cmake b/cime_config/machines/Depends.pm-gpu.gnugpu.cmake
@@ -0,0 +1,15 @@
+list(APPEND NO_INLINE_ARG_PACKING_LIST
+  eam/src/dynamics/se/inidat.F90
+)
+
+if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10)
+
+  if (NOT DEBUG)
+    # new in gnu10, inline arg packing was causing INF values with SMS_P4x1.ne4pg2_ne4pg2.F-MMFXX
+    foreach(ITEM IN LISTS NO_INLINE_ARG_PACKING_LIST)
+      e3sm_add_flags("${ITEM}" " -fno-inline-arg-packing")
+    endforeach()
+  endif()
+
+endif()
+
diff --git a/cime_config/machines/cmake_macros/gnu_perlmutter.cmake b/cime_config/machines/cmake_macros/gnu_perlmutter.cmake
diff --git a/cime_config/machines/cmake_macros/gnugpu_perlmutter.cmake b/cime_config/machines/cmake_macros/gnugpu_perlmutter.cmake
diff --git a/cime_config/machines/cmake_macros/nvidia.cmake b/cime_config/machines/cmake_macros/nvidia.cmake
@@ -5,35 +5,32 @@ set(SCC "nvcc")
 set(SCXX "nvc++")
 set(SFC "nvfortran")
 string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRNVIDIA")
-if (compile_threaded)
-  string(APPEND CFLAGS " -mp")
-endif()
 if (NOT DEBUG)
   string(APPEND CFLAGS " -O0 -Mnofma -Mvect=nosimd")
 endif()
 if (DEBUG)
   string(APPEND CFLAGS " -g")
 endif()
 string(APPEND FFLAGS " -i4 -Mstack_arrays  -Mextend -byteswapio -Mflushz -Kieee -DHAVE_IEEE_ARITHMETIC -Mallocatable=03 -DNO_R16 -traceback")
-if (compile_threaded)
-  string(APPEND FFLAGS " -mp")
-endif()
 if (DEBUG)
   string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee")
 endif()
 if (NOT DEBUG)
   string(APPEND FFLAGS " -O2")
 endif()
 string(APPEND CXXFLAGS " -std=c++14")
-if (compile_threaded)
-  string(APPEND CXXFLAGS " -mp")
-endif()
 if (DEBUG)
   string(APPEND CXXFLAGS " -g -O0 -Mnofma -Wall -Kieee -traceback")
 endif()
 if (NOT DEBUG)
   string(APPEND CXXFLAGS " -O2")
 endif()
+if (compile_threaded)
+  string(APPEND FFLAGS " -mp")
+  string(APPEND CFLAGS " -mp")
+  string(APPEND CXXFLAGS " -mp")
+  string(APPEND LDFLAGS " -mp")
+endif()
 set(CXX_LINKER "CXX")
 string(APPEND CXX_LIBS " -lstdc++")
 string(APPEND FC_AUTO_R8 " -r8")
@@ -42,7 +39,4 @@ string(APPEND FIXEDFLAGS " -Mfixed")
 string(APPEND FREEFLAGS " -Mfree")
 set(HAS_F2008_CONTIGUOUS "FALSE")
 string(APPEND LDFLAGS " -Wl,--allow-multiple-definition")
-if (compile_threaded)
-  string(APPEND LDFLAGS " -mp")
-endif()
 set(SUPPORTS_CXX "TRUE")
diff --git a/cime_config/machines/cmake_macros/nvidia_perlmutter.cmake b/cime_config/machines/cmake_macros/nvidia_perlmutter.cmake
diff --git a/cime_config/machines/cmake_macros/nvidia_pm-cpu.cmake b/cime_config/machines/cmake_macros/nvidia_pm-cpu.cmake
@@ -20,7 +20,8 @@ if (NOT DEBUG)
   string(APPEND FFLAGS " -O2")
 endif()
 if (compile_threaded)
-  string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_OPENMP=Off") # work-around for nvidia as kokkos is not passing "-mp" for threaded build
+  #set(KOKKOS_OPTIONS "--with-serial --with-openmp")
+  set(KOKKOS_OPTIONS "--with-serial") # work-around for nvidia as kokkos is not passing "-mp" for threaded build
 endif()
 set(MPICC "cc")
 set(MPICXX "CC")

diff --git a/cime_config/machines/cmake_macros/nvidia_pm-gpu.cmake b/cime_config/machines/cmake_macros/nvidia_pm-gpu.cmake
@@ -20,7 +20,7 @@ if (NOT DEBUG)
   string(APPEND FFLAGS " -O2")
 endif()
 if (compile_threaded)
-  string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_OPENMP=Off") # work-around for nvidia as kokkos is not passing "-mp" for threaded build
+  set(KOKKOS_OPTIONS "--with-serial") # work-around for nvidia as kokkos is not passing "-mp" for threaded build
 endif()
 set(MPICC "cc")
 set(MPICXX "CC")

diff --git a/cime_config/machines/cmake_macros/nvidiagpu_perlmutter.cmake b/cime_config/machines/cmake_macros/nvidiagpu_perlmutter.cmake
diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml
@@ -334,26 +334,48 @@
     </queues>
   </batch_system>
 
-  <batch_system MACH="perlmutter" type="nersc_slurm">
+  <batch_system MACH="miller" type="miller_slurm">
+    <queues>
+      <queue walltimemax="06:00:00" nodemax="800" default="true">collaboration</queue>
+    </queues>
+  </batch_system>
+
+  <batch_system MACH="pm-gpu" type="nersc_slurm">
     <directives>
       <directive> --constraint=gpu</directive>
     </directives>
-    <directives compiler="nvidiagpu">
+    <directives compiler="gnugpu">
       <directive> --gpus-per-task=1</directive>
-      <directive> --gpu-bind=map_gpu:0,1,2,3</directive>
     </directives>
-    <directives compiler="nvidia">
-      <directive> -G 0</directive>
+    <directives COMPSET="!.*MMF.*" compiler="gnugpu">
+      <directive> --gpu-bind=none</directive>
     </directives>
-    <directives compiler="gnugpu">
-      <directive> --gpus-per-task=1</directive>
+    <directives COMPSET=".*MMF.*" compiler="gnugpu">
       <directive> --gpu-bind=map_gpu:0,1,2,3</directive>
     </directives>
+    <directives compiler="nvidiagpu">
+      <directive> --gpus-per-task=1</directive>
+      <directive> --gpu-bind=none</directive>
+    </directives>
     <directives compiler="gnu">
       <directive> -G 0</directive>
     </directives>
+    <directives compiler="nvidia">
+      <directive> -G 0</directive>
+    </directives>
     <queues>
       <queue walltimemax="00:45:00" nodemax="1500" default="true">regular</queue>
+      <queue walltimemax="00:15:00" nodemax="4" strict="true">debug</queue>
+    </queues>
+  </batch_system>
+
+  <batch_system MACH="pm-cpu" type="nersc_slurm">
+    <directives>
+      <directive> --constraint=cpu</directive>
+    </directives>
+    <queues>
+      <queue walltimemax="00:30:00" nodemax="1500" default="true">regular</queue>
+      <queue walltimemax="00:30:00" nodemax="4" strict="true">debug</queue>
     </queues>
   </batch_system>