Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

v11: Update Intel MPI flags #652

Merged
merged 1 commit into from
Oct 18, 2024
Merged

v11: Update Intel MPI flags #652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 20 additions & 37 deletions gcm_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,17 @@ setenv ARCH `uname -s`
setenv NODE `uname -n`

if ($ARCH == Darwin) then
set FINDPATH = realpath
set PRELOAD_COMMAND = 'DYLD_INSERT_LIBRARIES'
set LD_LIBRARY_PATH_CMD = 'DYLD_LIBRARY_PATH'
# On macOS we seem to need to call mpirun directly and not use esma_mpirun
# For some reason SIP does not let the libraries be preloaded
set RUN_CMD = 'mpirun -np '
else
set FINDPATH = 'readlink -f'
set PRELOAD_COMMAND = 'LD_PRELOAD'
set LD_LIBRARY_PATH_CMD = 'LD_LIBRARY_PATH'
set RUN_CMD = '$GEOSBIN/esma_mpirun -np '
endif
set GCMSETUP = `$FINDPATH $0`
set BINDIR = `dirname $GCMSETUP`
set BINDIR = `pwd -L`
set GEOSDEF = `dirname $BINDIR`
set ETCDIR = ${GEOSDEF}/etc

Expand Down Expand Up @@ -1815,7 +1812,7 @@ else if( $SITE == 'AWS' | $SITE == 'Azure' ) then

setenv SSTDIR ${BOUNDARY_DIR}/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /home/$LOGNAME # user work directory
setenv WRKDIR $HOME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files

Expand Down Expand Up @@ -1853,7 +1850,7 @@ else

setenv SSTDIR ${BOUNDARY_DIR}/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /home/$LOGNAME # user work directory
setenv WRKDIR $HOME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files
set NX = 1
Expand Down Expand Up @@ -2188,9 +2185,6 @@ endif
else if( $MPI_STACK == intelmpi ) then

cat > $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

# This flag prints out the Intel MPI state. Uncomment if needed
#setenv I_MPI_DEBUG 9
EOF
Expand All @@ -2200,7 +2194,24 @@ EOF
# specific compared to the above adjustments
if ( $SITE == 'NCCS' ) then

# Some flags we know work on SLES15 and Milan (see below)
# For safety, we keep the old SLES12 flags for that system
#
# NOTE: When Cascade Lake is on SLES15, the following flags
# might need to be Milan only

if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3
EOF

else

cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_SHM_HEAP_VSIZE 512
setenv PSM2_MEMORY large
EOF
Expand All @@ -2216,34 +2227,6 @@ EOF

endif # if NOT Singularity

# Testing by Bill Putman found these to be
# useful flags with Intel MPI on SLES15 on the
# Milan nodes.
# Note 1: Testing by NCCS shows the PSM3 provider
# runs on the Infiniband fabric. Tests show it runs
# up to C720.
# Note 2: When the Cascade Lakes are moved to
# SLES15, these will need to be Milan-only flags
# as Intel MPI will probably work just fine with
# Intel chips.
if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FALLBACK 0
setenv I_MPI_FABRICS ofi
setenv I_MPI_OFI_PROVIDER psm3
setenv I_MPI_ADJUST_SCATTER 2
setenv I_MPI_ADJUST_SCATTERV 2
setenv I_MPI_ADJUST_GATHER 2
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_ADJUST_ALLGATHER 3
setenv I_MPI_ADJUST_ALLGATHERV 3
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_REDUCE 10
setenv I_MPI_ADJUST_BCAST 11
setenv I_MPI_ADJUST_REDUCE_SCATTER 4
setenv I_MPI_ADJUST_BARRIER 9
EOF

endif # if SLES15

endif # if NCCS
Expand Down
61 changes: 22 additions & 39 deletions geoschemchem_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,17 @@ setenv ARCH `uname -s`
setenv NODE `uname -n`

if ($ARCH == Darwin) then
set FINDPATH = realpath
set PRELOAD_COMMAND = 'DYLD_INSERT_LIBRARIES'
set LD_LIBRARY_PATH_CMD = 'DYLD_LIBRARY_PATH'
# On macOS we seem to need to call mpirun directly and not use esma_mpirun
# For some reason SIP does not let the libraries be preloaded
set RUN_CMD = 'mpirun -np '
else
set FINDPATH = 'readlink -f'
set PRELOAD_COMMAND = 'LD_PRELOAD'
set LD_LIBRARY_PATH_CMD = 'LD_LIBRARY_PATH'
set RUN_CMD = '$GEOSBIN/esma_mpirun -np '
endif
set GCMSETUP = `$FINDPATH $0`
set BINDIR = `dirname $GCMSETUP`
set BINDIR = `pwd -L`
set GEOSDEF = `dirname $BINDIR`
set ETCDIR = ${GEOSDEF}/etc

Expand Down Expand Up @@ -1763,7 +1760,7 @@ if( $SITE == 'NAS' ) then
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /nobackup/$LOGNAME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files

else if( $SITE == 'NCCS' ) then
setenv BATCH_CMD "sbatch" # SLURM Batch command
Expand Down Expand Up @@ -1839,13 +1836,13 @@ else if( $SITE == 'AWS' | $SITE == 'Azure' ) then
setenv BOUNDARY_DIR /ford1/share/gmao_SIteam/ModelData
setenv BC_BASE ${BOUNDARY_DIR}/bcs_shared/fvInput/ExtData/esm/tiles
setenv BCSDIR ${BC_BASE}/${LSM_BCS} # location of Boundary Conditions
setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY
setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY
setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY
setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY

setenv SSTDIR ${BOUNDARY_DIR}/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /home/$LOGNAME # user work directory
setenv WRKDIR $HOME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files

Expand Down Expand Up @@ -1883,7 +1880,7 @@ else

setenv SSTDIR ${BOUNDARY_DIR}/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /home/$LOGNAME # user work directory
setenv WRKDIR $HOME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files
set NX = 1
Expand Down Expand Up @@ -2218,9 +2215,6 @@ endif
else if( $MPI_STACK == intelmpi ) then

cat > $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

# This flag prints out the Intel MPI state. Uncomment if needed
#setenv I_MPI_DEBUG 9
EOF
Expand All @@ -2230,7 +2224,24 @@ EOF
# specific compared to the above adjustments
if ( $SITE == 'NCCS' ) then

# Some flags we know work on SLES15 and Milan (see below)
# For safety, we keep the old SLES12 flags for that system
#
# NOTE: When Cascade Lake is on SLES15, the following flags
# might need to be Milan only

if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3
EOF

else

cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_SHM_HEAP_VSIZE 512
setenv PSM2_MEMORY large
EOF
Expand All @@ -2246,34 +2257,6 @@ EOF

endif # if NOT Singularity

# Testing by Bill Putman found these to be
# useful flags with Intel MPI on SLES15 on the
# Milan nodes.
# Note 1: Testing by NCCS shows the PSM3 provider
# runs on the Infiniband fabric. Tests show it runs
# up to C720.
# Note 2: When the Cascade Lakes are moved to
# SLES15, these will need to be Milan-only flags
# as Intel MPI will probably work just fine with
# Intel chips.
if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FALLBACK 0
setenv I_MPI_FABRICS ofi
setenv I_MPI_OFI_PROVIDER psm3
setenv I_MPI_ADJUST_SCATTER 2
setenv I_MPI_ADJUST_SCATTERV 2
setenv I_MPI_ADJUST_GATHER 2
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_ADJUST_ALLGATHER 3
setenv I_MPI_ADJUST_ALLGATHERV 3
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_REDUCE 10
setenv I_MPI_ADJUST_BCAST 11
setenv I_MPI_ADJUST_REDUCE_SCATTER 4
setenv I_MPI_ADJUST_BARRIER 9
EOF

endif # if SLES15

endif # if NCCS
Expand Down
63 changes: 23 additions & 40 deletions gmichem_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,17 @@ setenv ARCH `uname -s`
setenv NODE `uname -n`

if ($ARCH == Darwin) then
set FINDPATH = realpath
set PRELOAD_COMMAND = 'DYLD_INSERT_LIBRARIES'
set LD_LIBRARY_PATH_CMD = 'DYLD_LIBRARY_PATH'
# On macOS we seem to need to call mpirun directly and not use esma_mpirun
# For some reason SIP does not let the libraries be preloaded
set RUN_CMD = 'mpirun -np '
else
set FINDPATH = 'readlink -f'
set PRELOAD_COMMAND = 'LD_PRELOAD'
set LD_LIBRARY_PATH_CMD = 'LD_LIBRARY_PATH'
set RUN_CMD = '$GEOSBIN/esma_mpirun -np '
endif
set GCMSETUP = `$FINDPATH $0`
set BINDIR = `dirname $GCMSETUP`
set BINDIR = `pwd -L`
set GEOSDEF = `dirname $BINDIR`
set ETCDIR = ${GEOSDEF}/etc

Expand Down Expand Up @@ -1888,7 +1885,7 @@ if( $SITE == 'NAS' ) then
setenv BATCH_TIME "PBS -l walltime=" # PBS Syntax for walltime
setenv BATCH_JOBNAME "PBS -N " # PBS Syntax for job name
setenv BATCH_OUTPUTNAME "PBS -o " # PBS Syntax for job output name
setenv BATCH_JOINOUTERR "PBS -j oe -k oed" # PBS Syntax for joining output and error
setenv BATCH_JOINOUTERR "PBS -j oe -k oed" # PBS Syntax for joining output and error
setenv RUN_FT "6:00:00" # Wallclock Time for gcm_forecast.j
setenv RUN_T "8:00:00" # Wallclock Time for gcm_run.j
setenv POST_T "8:00:00" # Wallclock Time for gcm_post.j
Expand Down Expand Up @@ -1928,7 +1925,7 @@ if( $SITE == 'NAS' ) then
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /nobackup/$LOGNAME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files

else if( $SITE == 'NCCS' ) then
setenv BATCH_CMD "sbatch" # SLURM Batch command
Expand Down Expand Up @@ -2004,13 +2001,13 @@ else if( $SITE == 'AWS' | $SITE == 'Azure' ) then
setenv BOUNDARY_DIR /ford1/share/gmao_SIteam/ModelData
setenv BC_BASE ${BOUNDARY_DIR}/bcs_shared/fvInput/ExtData/esm/tiles
setenv BCSDIR ${BC_BASE}/${LSM_BCS} # location of Boundary Conditions
setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY
setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY
setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY
setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY

setenv SSTDIR ${BOUNDARY_DIR}/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /home/$LOGNAME # user work directory
setenv WRKDIR $HOME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files

Expand Down Expand Up @@ -2048,7 +2045,7 @@ else

setenv SSTDIR ${BOUNDARY_DIR}/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /home/$LOGNAME # user work directory
setenv WRKDIR $HOME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files
set NX = 1
Expand Down Expand Up @@ -2389,9 +2386,6 @@ endif
else if( $MPI_STACK == intelmpi ) then

cat > $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

# This flag prints out the Intel MPI state. Uncomment if needed
#setenv I_MPI_DEBUG 9
EOF
Expand All @@ -2401,7 +2395,24 @@ EOF
# specific compared to the above adjustments
if ( $SITE == 'NCCS' ) then

# Some flags we know work on SLES15 and Milan (see below)
# For safety, we keep the old SLES12 flags for that system
#
# NOTE: When Cascade Lake is on SLES15, the following flags
# might need to be Milan only

if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3
EOF

else

cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_SHM_HEAP_VSIZE 512
setenv PSM2_MEMORY large
EOF
Expand All @@ -2417,34 +2428,6 @@ EOF

endif # if NOT Singularity

# Testing by Bill Putman found these to be
# useful flags with Intel MPI on SLES15 on the
# Milan nodes.
# Note 1: Testing by NCCS shows the PSM3 provider
# runs on the Infiniband fabric. Tests show it runs
# up to C720.
# Note 2: When the Cascade Lakes are moved to
# SLES15, these will need to be Milan-only flags
# as Intel MPI will probably work just fine with
# Intel chips.
if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FALLBACK 0
setenv I_MPI_FABRICS ofi
setenv I_MPI_OFI_PROVIDER psm3
setenv I_MPI_ADJUST_SCATTER 2
setenv I_MPI_ADJUST_SCATTERV 2
setenv I_MPI_ADJUST_GATHER 2
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_ADJUST_ALLGATHER 3
setenv I_MPI_ADJUST_ALLGATHERV 3
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_REDUCE 10
setenv I_MPI_ADJUST_BCAST 11
setenv I_MPI_ADJUST_REDUCE_SCATTER 4
setenv I_MPI_ADJUST_BARRIER 9
EOF

endif # if SLES15

endif # if NCCS
Expand Down
Loading
Loading