From d45ea772c8d477a4be900a770ce831159a65b53d Mon Sep 17 00:00:00 2001
From: Anton Steketee <79179784+anton-seaice@users.noreply.github.com>
Date: Tue, 30 Apr 2024 10:55:30 +1000
Subject: [PATCH] Standalone gadi (#2)

* Standalone gadi setup (#1)
---
 configuration/scripts/cice.batch.csh          | 25 +++++++
 configuration/scripts/cice.launch.csh         | 12 +++
 .../scripts/machines/Macros.gadi_intel        | 73 +++++++++++++++++++
 configuration/scripts/machines/env.gadi_intel | 56 ++++++++++++++
 4 files changed, 166 insertions(+)
 create mode 100644 configuration/scripts/machines/Macros.gadi_intel
 create mode 100644 configuration/scripts/machines/env.gadi_intel

diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh
index 50ef665bd..520d165a3 100755
--- a/configuration/scripts/cice.batch.csh
+++ b/configuration/scripts/cice.batch.csh
@@ -48,6 +48,31 @@ cat >> ${jobfile} << EOFB
 ###PBS -m be
 EOFB
 
+else if (${ICE_MACHINE} =~ gadi*) then
+if (${queue} =~ *sr) then #sapphire rapids
+  @ memuse = ( $ncores * 481 / 100 )
+else if (${queue} =~ *bw) then #broadwell
+  @ memuse = ( $ncores * 457 / 100 )
+else if (${queue} =~ *sl) then 
+  @ memuse = ( $ncores * 6 )
+else #normal queues
+  @ memuse = ( $ncores * 395 / 100 )
+endif
+cat >> ${jobfile} << EOFB
+#PBS -q ${queue}
+#PBS -P ${ICE_MACHINE_PROJ}
+#PBS -N ${ICE_CASENAME}
+#PBS -l storage=gdata/${ICE_MACHINE_PROJ}+scratch/${ICE_MACHINE_PROJ}+gdata/ik11
+#PBS -l ncpus=${ncores}
+#PBS -l mem=${memuse}gb
+#PBS -l walltime=${batchtime}
+#PBS -j oe 
+#PBS -W umask=003
+#PBS -o ${ICE_CASEDIR}
+source /etc/profile.d/modules.csh
+module use `echo ${MODULEPATH} | sed 's/:/ /g'` #copy the users modules
+EOFB
+
 else if (${ICE_MACHINE} =~ gust*) then
 cat >> ${jobfile} << EOFB
 #PBS -q ${queue}
diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh
index 51c8f044f..f8eb0a5d2 100755
--- a/configuration/scripts/cice.launch.csh
+++ b/configuration/scripts/cice.launch.csh
@@ -46,6 +46,18 @@ mpiexec --cpu-bind depth -n ${ntasks} -ppn ${taskpernodelimit} -d ${nthrds} ./ci
 EOFR
 endif
 
+#=======
+else if (${ICE_MACHCOMP} =~ gadi*) then
+if (${ICE_COMMDIR} =~ serial*) then
+cat >> ${jobfile} << EOFR
+./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+else
+cat >> ${jobfile} << EOFR
+mpirun -n ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+endif
+
 #=======
 else if (${ICE_MACHCOMP} =~ hobart* || ${ICE_MACHCOMP} =~ izumi*) then
 if (${ICE_COMMDIR} =~ serial*) then
diff --git a/configuration/scripts/machines/Macros.gadi_intel b/configuration/scripts/machines/Macros.gadi_intel
new file mode 100644
index 000000000..df7746731
--- /dev/null
+++ b/configuration/scripts/machines/Macros.gadi_intel
@@ -0,0 +1,73 @@
+#==============================================================================
+# Makefile macros for NCI Gadi, intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE -DREPRODUCIBLE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise -Wno-unused-variable -Wno-unused-parameter
+
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+
+NCI_INTEL_FLAGS := -r8 -i4 -traceback -w -fpe0 -ftz -convert big_endian -assume byterecl -check noarg_temp_created
+NCI_REPRO_FLAGS := -fp-model precise -fp-model source -align all
+
+ifeq ($(ICE_BLDDEBUG), true)
+    NCI_DEBUG_FLAGS := -g3 -O0 -debug all -check all -no-vec -assume nobuffered_io
+    FFLAGS          := $(NCI_INTEL_FLAGS) $(NCI_REPRO_FLAGS) $(NCI_DEBUG_FLAGS)
+    CPPDEFS         := $(CPPDEFS) -DDEBUG=$(DEBUG)
+else
+  NCI_OPTIM_FLAGS := -g3 -O2 -axCORE-AVX2 -debug all -check none -qopt-report=5 -qopt-report-annotate -assume buffered_io
+  FFLAGS          := $(NCI_INTEL_FLAGS) $(NCI_REPRO_FLAGS) $(NCI_OPTIM_FLAGS)
+endif
+
+SCC   := icx 
+SFC   := ifort
+MPICC := mpicc
+MPIFC := mpifort
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+SLIBS := $(SLIBS)
+INCLDIR := $(INCLDIR)
+
+# if spack modules loaded, use them. otherwise use system modules
+ifndef SPACK_NETCDF_FORTRAN_ROOT  
+   SLIBS += -L$(NETCDF)/lib -lnetcdf -lnetcdff
+   INCLDIR += -I$(NETCDF)/include
+else
+   SLIBS += -L$(SPACK_NETCDF_C_ROOT)/lib64 -lnetcdf
+   SLIBS += -L$(SPACK_NETCDF_FORTRAN_ROOT)/lib  -lnetcdff
+   INCLDIR += -I$(SPACK_NETCDF_C_ROOT)/include
+   INCLDIR += -I$(SPACK_NETCDF_FORTRAN_ROOT)/include
+endif
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp 
+   CFLAGS += -qopenmp 
+   FFLAGS += -qopenmp 
+endif
+
+ifeq ($(ICE_IOTYPE), pio1)
+   LIB_PIO := $(PIO_LIBDIR)
+   SLIBS   += -L$(LIB_PIO) -lpio
+endif
+
+ifeq ($(ICE_IOTYPE), pio2)
+  ifndef SPACK_PARALLELIO_ROOT
+    SLIBS += -L$(PARALLELIO_ROOT)/lib -lpioc -lpiof
+  else
+   SLIBS += -L$(SPACK_PARALLELIO_ROOT)/lib -lpioc -lpiof  
+   INCLDIR += -I $(SPACK_PARALLELIO_ROOT)/include
+  endif
+   
+  SLIBS += $(SLIBS) -L$(OMPI_BASE)/lib -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh 
+   
+endif
\ No newline at end of file
diff --git a/configuration/scripts/machines/env.gadi_intel b/configuration/scripts/machines/env.gadi_intel
new file mode 100644
index 000000000..ece74fc72
--- /dev/null
+++ b/configuration/scripts/machines/env.gadi_intel
@@ -0,0 +1,56 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+  source /etc/profile.d/modules.csh
+
+  module load intel-compiler
+  module load openmpi
+  
+  if ($?ICE_IOTYPE) then
+    if ($ICE_IOTYPE =~ pio*) then
+      if ($ICE_IOTYPE == "pio1") then
+        # we don't have pio1 installed anywhere  
+        module load pnetcdf
+        module load netcdf
+        module load pio
+      else
+        module load parallelio
+      endif
+    else 
+      module load netcdf
+    endif
+  endif
+
+  if ($?ICE_BFBTYPE) then
+    if ($ICE_BFBTYPE =~ qcchk*) then
+      # conda/analysis has the required librarys, skip building from cice yaml file
+      module use /g/data/hh5/public/modules
+      module load conda/analysis
+      # conda env create -f ../../configuration/scripts/tests/qctest.yml
+      # conda activate qctest
+    endif
+  endif
+
+endif
+
+setenv ICE_MACHINE_MACHNAME gadi
+setenv ICE_MACHINE_MACHINFO "Intel Xeon Scalable"
+setenv ICE_MACHINE_ENVNAME intel
+# setenv ICE_MACHINE_ENVINFO ${module list}
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR /scratch/$PROJECT/$USER/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /g/data/ik11/inputs
+setenv ICE_MACHINE_BASELINE /scratch/$PROJECT/$USER/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_PROJ $PROJECT
+setenv ICE_MACHINE_ACCT $USER
+setenv ICE_MACHINE_QUEUE "normal"
+setenv ICE_MACHINE_TPNODE 48
+setenv ICE_MACHINE_BLDTHRDS 4
+setenv ICE_MACHINE_QSTAT "qstat"