Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Upgrade the GSI to Spack-Stack version 1.6.0 #684

Merged
merged 20 commits into from
Feb 14, 2024
Merged

Upgrade the GSI to Spack-Stack version 1.6.0 #684

Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
c23b510
Initial upgrade to spack-stack v1.6.0. #674
DavidHuber-NOAA Jan 17, 2024
2136882
Update CI library versions.
DavidHuber-NOAA Jan 17, 2024
89d8bf8
Update CI library versions to SS versions.
DavidHuber-NOAA Jan 18, 2024
598c9f7
Merge remote-tracking branch 'origin/develop' into feature/ss_160
DavidHuber-NOAA Jan 18, 2024
955eedf
Remove Cheyenne.
DavidHuber-NOAA Jan 19, 2024
7f3969a
Remove Cheyenne submit script.
DavidHuber-NOAA Jan 19, 2024
339bf16
Fix gsi-addon paths for hercules, orion, and S4.
DavidHuber-NOAA Jan 22, 2024
2ae8c3c
Merge remote-tracking branch 'origin/develop' into feature/ss_160
DavidHuber-NOAA Jan 31, 2024
04a737d
Update Gaea-C5 support.
DavidHuber-NOAA Jan 31, 2024
bfb46e2
Remove references to Cheyenne.
DavidHuber-NOAA Jan 31, 2024
2efa4ba
Merge remote-tracking branch 'origin/develop' into feature/ss_160
DavidHuber-NOAA Jan 31, 2024
aac7010
Explicitly enable lustre parallel i/o.
DavidHuber-NOAA Feb 1, 2024
a906115
Revert Hercules modules to SS/1.5.1.
DavidHuber-NOAA Feb 9, 2024
7fda3e4
Merge remote-tracking branch 'origin/develop' into feature/ss_160
DavidHuber-NOAA Feb 9, 2024
6dd0413
Merge branch 'feature/ss_160' of github.com:davidhuber-noaa/gsi into …
DavidHuber-NOAA Feb 9, 2024
6e085fb
Merge remote-tracking branch 'origin/develop' into feature/ss_160
DavidHuber-NOAA Feb 12, 2024
4c923ab
Revert Gaea support.
DavidHuber-NOAA Feb 12, 2024
e56921c
Re-update Hercules to SS/1.6.0.
DavidHuber-NOAA Feb 13, 2024
3951289
Merge remote-tracking branch 'origin/develop' into feature/ss_160
DavidHuber-NOAA Feb 13, 2024
11a4ffd
Update openblas version for GNU/SS v1.6.0.
DavidHuber-NOAA Feb 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions ci/spack.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,19 @@ spack:
- gcc@10:10
specs:
- netcdf-c@4.9.2
- netcdf-fortran@4.6.0
- netcdf-fortran@4.6.1
- bufr@11.7.0
- bacio@2.4.1
- w3emc@2.9.2
- sp@2.3.3
- ip@3.3.3
- w3emc@2.10.0
- sp@2.5.0
- ip@4.3.0
- sigio@2.3.2
- sfcio@1.4.1
- nemsio@2.5.4
- wrf-io@1.2.0
- ncio@1.1.2
- crtm@2.4.0
- gsi-ncdiag@1.1.1
- crtm@2.4.0.1
- gsi-ncdiag@1.1.2
view: true
concretizer:
unify: true
32 changes: 0 additions & 32 deletions modulefiles/gsi_cheyenne.gnu.lua
View file
Open in desktop

This file was deleted.

32 changes: 0 additions & 32 deletions modulefiles/gsi_cheyenne.intel.lua
View file
Open in desktop

This file was deleted.

6 changes: 3 additions & 3 deletions modulefiles/gsi_common.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,19 @@ Load common modules to build GSI on all machines
]])

local netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0"
local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"

local bufr_ver=os.getenv("bufr_ver") or "11.7.0"
local bacio_ver=os.getenv("bacio_ver") or "2.4.1"
local w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
local sp_ver=os.getenv("sp_ver") or "2.3.3"
local sp_ver=os.getenv("sp_ver") or "2.5.0"
local ip_ver=os.getenv("ip_ver") or "4.3.0"
local sigio_ver=os.getenv("sigio_ver") or "2.3.2"
local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0"
local ncio_ver=os.getenv("ncio_ver") or "1.1.2"
local crtm_ver=os.getenv("crtm_ver") or "2.4.0"
local crtm_ver=os.getenv("crtm_ver") or "2.4.0.1"
local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.2"

load(pathJoin("netcdf-c", netcdf_c_ver))
Expand Down
6 changes: 3 additions & 3 deletions modulefiles/gsi_hera.gnu.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
help([[
]])

prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
--Needed for openmpi build
prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")

local python_ver=os.getenv("python_ver") or "3.10.8"
local python_ver=os.getenv("python_ver") or "3.11.6"
local stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0"
local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5"
local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"
local openblas_ver=os.getenv("openblas_ver") or "0.3.19"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe openblas version should be 0.3.24, @DavidHuber-NOAA can you confirm?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AlexanderRichert-NOAA You are correct, it should be 0.3.24. I updated it and ran a test build. Thanks!


load(pathJoin("stack-gcc", stack_gnu_ver))
Expand Down
6 changes: 3 additions & 3 deletions modulefiles/gsi_hera.intel.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
help([[
]])

prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")

local python_ver=os.getenv("python_ver") or "3.10.8"
local python_ver=os.getenv("python_ver") or "3.11.6"
local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"

load(pathJoin("stack-intel", stack_intel_ver))
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
Expand Down
6 changes: 3 additions & 3 deletions modulefiles/gsi_hercules.intel.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
help([[
]])

prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core")

local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8"
local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6"
local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"

load(pathJoin("stack-intel", stack_intel_ver))
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
Expand Down
6 changes: 3 additions & 3 deletions modulefiles/gsi_jet.intel.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
help([[
]])

prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")

local python_ver=os.getenv("python_ver") or "3.10.8"
local python_ver=os.getenv("python_ver") or "3.11.6"
local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"

load(pathJoin("stack-intel", stack_intel_ver))
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
Expand Down
6 changes: 3 additions & 3 deletions modulefiles/gsi_orion.intel.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
help([[
]])

prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core")

local stack_python_ver=os.getenv("python_ver") or "3.10.8"
local stack_python_ver=os.getenv("python_ver") or "3.11.6"
local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"

load(pathJoin("stack-intel", stack_intel_ver))
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
Expand Down
6 changes: 3 additions & 3 deletions modulefiles/gsi_s4.intel.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
help([[
]])

prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core")

local python_ver=os.getenv("python_ver") or "3.10.8"
local python_ver=os.getenv("python_ver") or "3.11.6"
local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.0"
local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"

load(pathJoin("stack-intel", stack_intel_ver))
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
Expand Down
2 changes: 1 addition & 1 deletion modulefiles/gsi_wcoss2.intel.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0"
local ncio_ver=os.getenv("ncio_ver") or "1.1.2"
local crtm_ver=os.getenv("crtm_ver") or "2.4.0"
local crtm_ver=os.getenv("crtm_ver") or "2.4.0.1"
local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1"

load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver))
Expand Down
32 changes: 0 additions & 32 deletions regression/regression_param.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,6 @@ case $machine in
;;
Discover)
sub_cmd="sub_discover"
;;
Cheyenne)
sub_cmd="sub_cheyenne"
memnode=128
numcore=36
;;
*) # EXIT out for unresolved machine
echo "unknown $machine"
Expand Down Expand Up @@ -73,9 +68,6 @@ case $regtest in
elif [[ "$machine" = "Discover" ]]; then
topts[1]="0:30:00" ; popts[1]="48/2" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="60/3" ; ropts[2]="/2"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:10:00" ; popts[1]="18/8/" ; ropts[1]="/1"
topts[2]="0:10:00" ; popts[2]="18/10/" ; ropts[2]="/2"
Expand Down Expand Up @@ -106,9 +98,6 @@ case $regtest in
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1"
Expand Down Expand Up @@ -139,9 +128,6 @@ case $regtest in
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1"
Expand Down Expand Up @@ -171,9 +157,6 @@ case $regtest in
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1"
Expand Down Expand Up @@ -204,9 +187,6 @@ case $regtest in
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1"
Expand Down Expand Up @@ -237,9 +217,6 @@ case $regtest in
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:30:00" ; popts[1]="8/6/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="8/8/" ; ropts[2]="/1"
Expand Down Expand Up @@ -270,9 +247,6 @@ case $regtest in
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
Expand Down Expand Up @@ -342,12 +316,6 @@ elif [[ "$machine" = "Gaea" ]]; then
export MPI_BUFS_PER_HOST=256
export MPI_GROUP_MAX=256
export APRUN="srun --export=ALL --mpi=pmi2 -n \$ntasks"
elif [[ "$machine" = "Cheyenne" ]]; then
export OMP_STACKSIZE=1024M
export MPI_BUFS_PER_PROC=256
export MPI_BUFS_PER_HOST=256
export MPI_GROUP_MAX=256
export APRUN="mpirun -v -np \$ntasks"
elif [[ "$machine" = "wcoss2" ]]; then
export OMP_PLACES=cores
export OMP_STACKSIZE=2G
Expand Down
18 changes: 1 addition & 17 deletions regression/regression_var.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,7 @@ else
fi

# Determine the machine
if [[ -d /glade ]]; then # Cheyenne
export machine="Cheyenne"
elif [[ -d /scratch1 ]]; then # Hera
if [[ -d /scratch1 ]]; then # Hera
export machine="Hera"
elif [[ -d /mnt/lfs4 || -d /jetmon || -d /mnt/lfs1 ]]; then # Jet
export machine="Jet"
Expand Down Expand Up @@ -66,20 +64,6 @@ case $machine in
export check_resource="no"
export accnt="nggps_emc"
;;
Cheyenne)
export queue="regular"
export noscrub="/glade/scratch/$LOGNAME/noscrub"
export group="global"
if [[ "$cmaketest" = "false" ]]; then
export basedir="/glade/scratch/$LOGNAME"
fi
export ptmp="/glade/scratch/$LOGNAME/$ptmpName"

export casesdir="/glade/work/epicufsrt/contrib/GSI_data/CASES/regtest"

export check_resource="no"
export accnt="NRAL0032"
;;
wcoss2)
export local_or_default="${local_or_default:-/lfs/h2/emc/da/noscrub/$LOGNAME}"
if [ -d $local_or_default ]; then
Expand Down
7 changes: 0 additions & 7 deletions ush/module-setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,6 @@ elif [[ $MACHINE_ID = wcoss2 ]]; then
# We are on WCOSS2
module reset

elif [[ $MACHINE_ID = cheyenne* ]] ; then
# We are on NCAR Cheyenne
if ( ! eval module help > /dev/null 2>&1 ) ; then
source /glade/u/apps/ch/modulefiles/default/localinit/localinit.sh
fi
module purge

elif [[ $MACHINE_ID = stampede* ]] ; then
# We are on TACC Stampede
if ( ! eval module help > /dev/null 2>&1 ) ; then
Expand Down
Loading
Loading