Port to Casper (CICE-Consortium#496)

Port to NCAR's cluster, casper using the HTC nodes using standard cpus. This will support testing and development on another piece of hardware other than derecho and izumi and provides a good option for smaller test cases. Added intel, inteloneapi, and gnu compiler options.

The -check debug option is very sensitive in this version of Inteloneapi, so it has been turned off.

configuration/scripts/icepack.batch.csh

@@ -64,6 +64,21 @@ cat >> ${jobfile} << EOFB
 ###PBS -m be
 EOFB
 
+else if (${ICE_MACHINE} =~ casper*) then
+cat >> ${jobfile} << EOFB
+#PBS -q ${ICE_MACHINE_QUEUE}
+#PBS -l job_priority=regular
+#PBS -N ${ICE_CASENAME}
+#PBS -A ${acct}
+#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}
+#PBS -l walltime=${ICE_RUNLENGTH}
+#PBS -j oe
+#PBS -W umask=022
+#PBS -o ${ICE_CASEDIR}
+###PBS -M username@domain.com
+###PBS -m be
+EOFB
+
 else if (${ICE_MACHINE} =~ hobart*) then
 cat >> ${jobfile} << EOFB
 #PBS -j oe 

configuration/scripts/machines/Macros.casper_gnu

@@ -0,0 +1,47 @@
+#==============================================================================
+# Makefile macros for NCAR derecho, gnu compiler
+#==============================================================================
+
+CPP        := ftn -E
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c
+
+FIXEDFLAGS := -ffixed-line-length-132
+FREEFLAGS  := -ffree-form
+FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS   += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow --std f2008
+#  FFLAGS   += -O0 -g -fcheck=all -finit-real=snan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+  CFLAGS   += -O0
+endif
+
+SCC   := gcc
+SFC   := gfortran
+CC := $(SCC)
+FC := $(SFC)
+LD := $(FC)
+
+NETCDF_PATH := $(NETCDF)
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs
+
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib
+
+INCLDIR := $(INCLDIR)
+
+LIB_NETCDF := $(NETCDF)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true)
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+

configuration/scripts/machines/Macros.casper_intel

@@ -0,0 +1,49 @@
+#==============================================================================
+# Makefile macros for NCAR derecho, intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise -march=core-avx2
+
+FIXEDFLAGS := -fixed -132
+FREEFLAGS  := -free
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -march=core-avx2
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := icx
+SFC   := ifort
+CC := $(SCC)
+FC := $(SFC)
+LD := $(FC)
+
+NETCDF_PATH := $(NETCDF)
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs
+
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib
+
+INCLDIR := $(INCLDIR)
+
+LIB_NETCDF := $(NETCDF)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true)
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+

configuration/scripts/machines/Macros.casper_inteloneapi

@@ -0,0 +1,51 @@
+#==============================================================================
+# Makefile macros for NCAR derecho, inteloneapi compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise -march=core-avx2
+
+FIXEDFLAGS := -fixed -132
+FREEFLAGS  := -free
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -march=core-avx2
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
+#  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
+#   LDFLAGS += -check uninit
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := icx
+SFC   := ifx
+CC := $(SCC)
+FC := $(SFC)
+LD := $(FC)
+
+NETCDF_PATH := $(NETCDF)
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs
+
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib
+
+INCLDIR := $(INCLDIR)
+
+LIB_NETCDF := $(NETCDF)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true)
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+

configuration/scripts/machines/env.casper_gnu

@@ -0,0 +1,51 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module --force purge
+module load ncarenv/23.10
+#module load craype
+module load gcc/12.2.0
+module load ncarcompilers
+#module load cray-mpich/8.1.25
+#module load hdf5/1.12.2
+module load netcdf/4.9.2
+#module load cray-libsci/23.02.1.1
+
+# For perftools with mpiexec
+# module load perftools-base
+# module load perftools
+#setenv PALS_TRANSFER FALSE
+
+endif
+
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv PALS_QUIET TRUE
+
+# May be needed for OpenMP memory
+setenv OMP_STACKSIZE 64M
+# OMP runtime diagnostics
+#setenv OMP_DISPLAY_ENV TRUE
+
+setenv ICE_MACHINE_MACHNAME casper
+setenv ICE_MACHINE_MACHINFO "NCAR cluster with Intel Cascade Lake 36 core nodes with Mellanox ConnectX-5"
+setenv ICE_MACHINE_ENVNAME gnu
+setenv ICE_MACHINE_ENVINFO "gcc 12.2.0 20220819, netcdf4.9.2"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
+setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
+setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "casper"
+setenv ICE_MACHINE_TPNODE 36
+setenv ICE_MACHINE_BLDTHRDS 1
+setenv ICE_MACHINE_QSTAT "qstat "

configuration/scripts/machines/env.casper_intel

@@ -0,0 +1,51 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module --force purge
+module load ncarenv/23.10
+#module load craype
+module load intel/2023.2.1
+module load ncarcompilers
+#module load cray-mpich/8.1.25
+#module load hdf5/1.12.2
+module load netcdf/4.9.2
+#module load cray-libsci/23.02.1.1
+
+# For perftools with mpiexec
+# module load perftools-base
+# module load perftools
+#setenv PALS_TRANSFER FALSE
+
+endif
+
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv PALS_QUIET TRUE
+
+# May be needed for OpenMP memory
+setenv OMP_STACKSIZE 64M
+# OMP runtime diagnostics
+#setenv OMP_DISPLAY_ENV TRUE
+
+setenv ICE_MACHINE_MACHNAME casper
+setenv ICE_MACHINE_MACHINFO "NCAR cluster with Intel Cascade Lake 36 core nodes with Mellanox ConnectX-5"
+setenv ICE_MACHINE_ENVNAME intel
+setenv ICE_MACHINE_ENVINFO "ifort 2021.10.0 20230609, netcdf4.9.2"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
+setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
+setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "casper"
+setenv ICE_MACHINE_TPNODE 36
+setenv ICE_MACHINE_BLDTHRDS 1
+setenv ICE_MACHINE_QSTAT "qstat "

configuration/scripts/machines/env.casper_inteloneapi

@@ -0,0 +1,51 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module --force purge
+module load ncarenv/23.10
+#module load craype
+module load intel-oneapi/2023.2.1
+module load ncarcompilers
+#module load cray-mpich/8.1.25
+#module load hdf5/1.12.2
+module load netcdf/4.9.2
+#module load cray-libsci/23.02.1.1
+
+# For perftools with mpiexec
+# module load perftools-base
+# module load perftools
+#setenv PALS_TRANSFER FALSE
+
+endif
+
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv PALS_QUIET TRUE
+
+# May be needed for OpenMP memory
+setenv OMP_STACKSIZE 64M
+# OMP runtime diagnostics
+#setenv OMP_DISPLAY_ENV TRUE
+
+setenv ICE_MACHINE_MACHNAME casper
+setenv ICE_MACHINE_MACHINFO "NCAR cluster with Intel Cascade Lake 36 core nodes with Mellanox ConnectX-5"
+setenv ICE_MACHINE_ENVNAME inteloneapi
+setenv ICE_MACHINE_ENVINFO "oneAPI DPC++/C++/ifx 2023.2.0 20230721, netcdf4.9.2"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
+setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
+setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "casper"
+setenv ICE_MACHINE_TPNODE 36
+setenv ICE_MACHINE_BLDTHRDS 1
+setenv ICE_MACHINE_QSTAT "qstat "