From 858e0dfb9509d3fd2d3fe5e48bb9927e4360bb6a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 21 Apr 2023 10:23:26 -0600 Subject: [PATCH] Update release/1.3.1 (starting from release/1.3.0) (#549) * Update .gitmodules and submodule pointer for spack for code review and testing * Update many versions in configs/common/packages.yaml and configs/templates/skylab-dev/spack.yaml, and remove ufs-weather-model~debug from configs/templates/unified-dev/spack.yaml and configs/templates/ufs-weather-model/spack.yaml * Update containers for new versions in release 1.3.1 * Replace gsibec@1.1.1 with gsibec@1.1.2 (bug fix) * Update submodule pointer for spack * Missing updates for containers * Update .gitmodules and container configs * More documentation updates * More documentation updates * Add 'spack clean -a' to github CI workflows * Add more 'spack clean -a' to github CI workflows * Remove useless 'spack clean -a' from CI tests * Update configs/containers/README.md * Add missing jedi-mpas-env to configs/templates/unified-dev/spack.yaml * Update .gitmodules and submodule pointer for spack * Finalize container config and documentation --- .github/workflows/macos-ci-x86_64.yaml | 1 + .github/workflows/ubuntu-ci-x86_64.yaml | 1 + .gitmodules | 2 +- configs/common/packages.yaml | 16 +- configs/containers/README.md | 35 +++- .../containers/docker-ubuntu-clang-mpich.yaml | 2 +- .../containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- .../containers/docker-ubuntu-intel-impi.yaml | 2 +- configs/templates/skylab-dev/spack.yaml | 17 +- .../templates/skylab-no-python-dev/spack.yaml | 94 ----------- .../templates/ufs-weather-model/spack.yaml | 1 - configs/templates/unified-dev/spack.yaml | 4 +- doc/source/PreConfiguredSites.rst | 156 +++++++++++++++++- spack | 2 +- 14 files changed, 203 insertions(+), 132 deletions(-) delete mode 100644 configs/templates/skylab-no-python-dev/spack.yaml diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index c1e842ab9..0057045eb 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -46,6 +46,7 @@ jobs: - name: create-env run: | source ./setup.sh + # DH* 20230302 - to avoid using padded_length for build caches, # always build in the same environment so that the length of the # path doesn't change - see also other 'DH* 20230302' changes below diff --git a/.github/workflows/ubuntu-ci-x86_64.yaml b/.github/workflows/ubuntu-ci-x86_64.yaml index 664e446f4..83609362f 100644 --- a/.github/workflows/ubuntu-ci-x86_64.yaml +++ b/.github/workflows/ubuntu-ci-x86_64.yaml @@ -46,6 +46,7 @@ jobs: - name: create-env run: | source ./setup.sh + # DH* 20230302 - to avoid using padded_length for build caches, # always build in the same environment so that the length of the # path doesn't change - see also other 'DH* 20230302' changes below diff --git a/.gitmodules b/.gitmodules index dae38f8b7..5e5979fba 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "spack"] path = spack url = https://github.com/NOAA-EMC/spack - branch = spack-stack-1.3.0 + branch = spack-stack-1.3.1 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index ff5622bc1..e672877ca 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -35,17 +35,17 @@ version: [2.4.0] variants: +fix ecbuild: - version: [3.6.5] + version: [3.7.2] eccodes: version: [2.27.0] ecflow: version: [5.8.4] variants: +ui eckit: - version: [1.20.2] + version: [1.23.0] variants: linalg=eigen,lapack compression=lz4,bzip2 ecmwf-atlas: - version: [0.32.1] + version: [0.33.0] variants: +fckit +trans ectrans: version: [1.2.0] @@ -62,7 +62,7 @@ version: [8.3.0b09] variants: ~xerces ~pnetcdf +pio snapshot=b09 ~shared fckit: - version: [0.10.0] + version: [0.10.1] variants: +eckit fftw: version: [3.3.10] @@ -90,14 +90,14 @@ grib-util: version: [1.2.3] gsibec: - version: [1.0.7] + version: [1.1.2] gsl-lite: version: [0.37.0] hdf: version: [4.2.15] variants: ~fortran ~netcdf hdf5: - version: [1.14.0] + version: [1.12.2] variants: +hl +fortran +mpi ~threadsafe +szip ip: version: [3.3.3] @@ -155,7 +155,7 @@ nlohmann-json-schema-validator: version: [2.1.0] odc: - version: [1.4.5] + version: [1.4.6] variants: ~fortran openblas: version: [0.3.19] @@ -184,7 +184,7 @@ py-eccodes: version: [1.4.2] py-h5py: - version: [3.7.0] + version: [3.6.0] variants: ~mpi # Comment out for now until build problems are solved # https://github.com/NOAA-EMC/spack-stack/issues/522 diff --git a/configs/containers/README.md b/configs/containers/README.md index 6b1bc6da9..1e11fbc02 100644 --- a/configs/containers/README.md +++ b/configs/containers/README.md @@ -2,20 +2,20 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs list empty (`specs: []`) and manually add the specs for the particular spack-stack release and application as listed below, *after* running `spack stack create ctr`. -### spack-stack-1.3.0 / skylab-4.0.0 containers for fv3-jedi and mpas-jedi (but not for ufs-jedi) +### spack-stack-1.3.1 / skylab-4.0.0 containers for fv3-jedi and mpas-jedi (but not for ufs-jedi) ``` specs: [base-env@1.0.0, jedi-base-env@1.0.0 ~fftw, jedi-ewok-env@1.0.0, jedi-fv3-env@1.0.0, - jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@11.7.1, ecbuild@3.6.5, eccodes@2.27.0, ecflow@5, - eckit@1.20.2, ecmwf-atlas@0.32.1 +trans ~fftw, ectrans@1.2.0 ~fftw, eigen@3.4.0, - fckit@0.10.0, fms@release-jcsda, g2@3.4.5, g2tmpl@1.10.0, gftl-shared@1.5.0, - gsibec@1.0.7, hdf@4.2.15, hdf5@1.14.0, ip@3.3.3, jasper@2.0.32, jedi-cmake@1.4.0, + jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@11.7.1, ecbuild@3.7.2, eccodes@2.27.0, ecflow@5, + eckit@1.23.0, ecmwf-atlas@0.33.0 +trans ~fftw, ectrans@1.2.0 ~fftw, eigen@3.4.0, + fckit@0.10.1, fms@release-jcsda, g2@3.4.5, g2tmpl@1.10.0, gftl-shared@1.5.0, + gsibec@1.1.2, hdf@4.2.15, hdf5@1.12.2, ip@3.3.3, jasper@2.0.32, jedi-cmake@1.4.0, libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.9.2, netcdf-cxx4@4.3.1, netcdf-fortran@4.6.0, nlohmann-json@3.10.5, nlohmann-json-schema-validator@2.1.0, parallelio@2.5.9, parallel-netcdf@1.12.2, py-eccodes@1.4.2, py-f90nml@1.4.3, - py-gitpython@3.1.27, py-numpy@1.22.3, + py-gitpython@3.1.27, py-h5py@3.6.0, py-numpy@1.22.3, py-pandas@1.4.0, py-pip, py-pyyaml@6.0, py-scipy@1.9.3, py-shapely@1.8.0, py-xarray@2022.3.0, sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, nco@5.0.6, - yafyaml@0.5.1, zlib@1.2.13, odc@1.4.5, crtm@v2.4-jedi.2, shumlib@macos_clang_linux_intel_port] + yafyaml@0.5.1, zlib@1.2.13, odc@1.4.6, crtm@v2.4.1-jedi, shumlib@macos_clang_linux_intel_port] # Don't build ESMF and MAPL for now: # https://github.com/JCSDA-internal/MPAS-Model/issues/38 # https://github.com/NOAA-EMC/spack-stack/issues/326 @@ -25,9 +25,28 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li # py-mysql-connector-python@8.0.32, ``` +### spack-stack-1.3.1 / ufs-weather-model-x.y.z containers for ufs-weather-model as of April 18, 2023 + +**Note. This is not yet working correctly, some libraries are missing. Please do not use yet! Also, if using the clang-mpich container, need to disable openmp for fms, not clear how to do this cleanly.** + +``` + specs: [base-env@1.0.0, + bacio@2.4.1, bison@3.8.2, bufr@11.7.1, ecbuild@3.7.2, eccodes@2.27.0, ecflow@5, + eckit@1.23.0, ecmwf-atlas@0.33.0 +trans ~fftw, ectrans@1.2.0 ~fftw, eigen@3.4.0, + fckit@0.10.1, fms@2022.04, g2@3.4.5, g2tmpl@1.10.0, gftl-shared@1.5.0, + gsibec@1.1.2, hdf@4.2.15, hdf5@1.12.2, ip@3.3.3, jasper@2.0.32, jedi-cmake@1.4.0, + libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.9.2, netcdf-cxx4@4.3.1, + netcdf-fortran@4.6.0, nlohmann-json@3.10.5, nlohmann-json-schema-validator@2.1.0, + parallelio@2.5.9, parallel-netcdf@1.12.2, py-eccodes@1.4.2, py-f90nml@1.4.3, + py-gitpython@3.1.27, py-h5py@3.6.0, py-numpy@1.22.3, py-pandas@1.4.0, + py-pip, py-pyyaml@6.0, py-scipy@1.9.3, py-shapely@1.8.0, py-xarray@2022.3.0, + sp@2.3.3, udunits@2.2.28, w3emc@2.9.2, w3nco@2.4.1, nco@5.0.6, esmf@8.3.0b09, mapl@2.22.0, + yafyaml@0.5.1, zlib@1.2.13, odc@1.4.6, crtm@2.4.0, crtm-fix@2.4.0_emc] +``` + ### Create an AMI on AWS EC2 to build docker containers -AMI ami-0b5951b8be1fe5709 (dom-docker-builder-full-backup-20221221; JCSDA-NOAA account, us-east-1) was created following the instructions below +AMI ami-0a934b7133c2b7102 (dom-docker-builder-full-backup-20230217; JCSDA-NOAA account, us-east-1) was created following the instructions below - See https://docs.docker.com/desktop/install/ubuntu/ - Start with ami-052efd3df9dad4825 diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index 57c4b909d..7a74f4f90 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -101,7 +101,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/noaa-emc/spack - ref: spack-stack-1.3.0 + ref: spack-stack-1.3.1 resolve_sha: false # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 1a1a7796d..ddc1ecc7e 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -88,7 +88,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/noaa-emc/spack - ref: spack-stack-1.3.0 + ref: spack-stack-1.3.1 resolve_sha: false # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 8df894b51..142aae9d6 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -106,7 +106,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/noaa-emc/spack - ref: spack-stack-1.3.0 + ref: spack-stack-1.3.1 resolve_sha: false # Whether or not to strip binaries diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 36895cdef..18b948b48 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -21,20 +21,20 @@ spack: - bufr@11.7.1 - crtm@2.3.0 - crtm@2.4.0 - - crtm@v2.3-jedi.4 - crtm@v2.4-jedi.2 + - crtm@v2.4.1-jedi - crtm@v3.0.0-rc.1 - - ecbuild@3.6.5 + - ecbuild@3.7.2 - eccodes@2.27.0 - ecflow@5 - - eckit@1.20.2 - - ecmwf-atlas@0.32.1 + - eckit@1.23.0 + - ecmwf-atlas@0.33.0 - ectrans@1.2.0 - eigen@3.4.0 - esmf@8.3.0b09 # DH* fake version number #- ewok@0.0.1 - - fckit@0.10.0 + - fckit@0.10.1 - fiat@1.1.0 # Do not request specific version of flex - this leads to duplicate packages being built #- flex@2.6.4 @@ -46,9 +46,9 @@ spack: #- gdal@3.4.3 #- geos@3.9.1 - gftl-shared@1.5.0 - - gsibec@1.0.7 + - gsibec@1.1.2 - hdf@4.2.15 - - hdf5@1.14.0 + - hdf5@1.12.2 - ip@3.3.3 - jasper@2.0.32 - jedi-cmake@1.4.0 @@ -62,11 +62,12 @@ spack: - netcdf-fortran@4.6.0 - nlohmann-json@3.10.5 - nlohmann-json-schema-validator@2.1.0 - - odc@1.4.5 + - odc@1.4.6 - parallelio@2.5.9 - parallel-netcdf@1.12.2 - py-eccodes@1.4.2 - py-f90nml@1.4.3 + - py-h5py@3.6.0 - py-numpy@1.22.3 - py-pandas@1.4.0 - py-pyyaml@6.0 diff --git a/configs/templates/skylab-no-python-dev/spack.yaml b/configs/templates/skylab-no-python-dev/spack.yaml deleted file mode 100644 index 7bdda82fb..000000000 --- a/configs/templates/skylab-no-python-dev/spack.yaml +++ /dev/null @@ -1,94 +0,0 @@ -spack: - concretizer: - unify: when_possible - - view: false - include: [] - - specs: - # Note. This template is provided as a convenience for people who prefer to use - # their own Python environments on top off non-Python libraries from spack-stack, - # and as a workaround for people who have problems compiling the Python packages - # in spack-stack on their system. - # We do not recommend using this template, and we do not officially support it. - # Use it at your own risk, and be aware that certain features may not work if - # dependencies used by Python packages (e.g. netcdf-c used by py-netcdf) are - # compiled with other options than what we need/what is in spack-stack. These - # inconsistencies in redundantly provided libraries are the main reason why - # spack builds those Python packages itself. - - # Virtual environment packages - # Explicitly turn off Python in jedi-base-env, - # this also requires removing jedi-ewok-env. - - jedi-base-env ~python - #- jedi-ewok-env - - jedi-fv3-env - - jedi-mpas-env - - jedi-ufs-env - - jedi-um-env - - soca-env - - # Individual packages - - bacio@2.4.1 - # Do not request specific version of bison - this leads to duplicate packages being built - #- bison@3.8.2 - - bufr@11.7.1 ~python - - crtm@2.3.0 - - crtm@2.4.0 - - crtm@v2.3-jedi.4 - - crtm@v2.4-jedi.2 - - crtm@v3.0.0-rc.1 - - ecbuild@3.6.5 - - eccodes@2.27.0 - # ecflow depends on Python - #- ecflow@5 - - eckit@1.20.2 - - ecmwf-atlas@0.32.1 - - ectrans@1.2.0 - - eigen@3.4.0 - - esmf@8.3.0b09 - # DH* fake version number - #- ewok@0.0.1 - - fckit@0.10.0 - - fiat@1.1.0 - # Do not request specific version of flex - this leads to duplicate packages being built - #- flex@2.6.4 - - fms@2022.04 - # DH* fake version number - - fms@release-jcsda - - g2@3.4.5 - - g2tmpl@1.10.2 - #- gdal@3.4.3 - #- geos@3.9.1 - - gftl-shared@1.5.0 - - gsibec@1.0.7 - - hdf@4.2.15 - - hdf5@1.14.0 - - ip@3.3.3 - - jasper@2.0.32 - - jedi-cmake@1.4.0 - - libpng@1.6.37 - - mapl@2.22.0 - - nccmp@1.9.0.1 - - ncview@2.1.8 - - nemsio@2.5.2 - - netcdf-c@4.9.2 - - netcdf-cxx4@4.3.1 - - netcdf-fortran@4.6.0 - - nlohmann-json@3.10.5 - - nlohmann-json-schema-validator@2.1.0 - - odc@1.4.5 - - parallelio@2.5.9 - - parallel-netcdf@1.12.2 - # DH* fake version number - #- r2d2@0.0.1 - # DH* fake version number - - shumlib@macos_clang_linux_intel_port - - sigio@2.3.2 - #- solo@1.0.0 - - sp@2.3.3 - - udunits@2.2.28 - - w3emc@2.9.2 - - w3nco@2.4.1 - - yafyaml@0.5.1 - - zlib@1.2.13 diff --git a/configs/templates/ufs-weather-model/spack.yaml b/configs/templates/ufs-weather-model/spack.yaml index def810b2c..5318ca2a2 100644 --- a/configs/templates/ufs-weather-model/spack.yaml +++ b/configs/templates/ufs-weather-model/spack.yaml @@ -16,4 +16,3 @@ spack: specs: - ufs-weather-model-env~debug - - ufs-weather-model-env+debug diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index f026cfb88..6634243e0 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -12,6 +12,7 @@ spack: #- gsi-env@unified-dev - jedi-ewok-env@unified-dev - jedi-fv3-env@unified-dev + - jedi-mpas-env@unified-dev - jedi-tools-env@unified-dev - jedi-ufs-env@unified-dev - jedi-um-env@unified-dev @@ -19,8 +20,7 @@ spack: - soca-env@unified-dev - ufs-srw-app-env@unified-dev #- ufs-utils-env@unified-dev - - ufs-weather-model-env~debug@unified-dev - - ufs-weather-model-env+debug@unified-dev + - ufs-weather-model-env@unified-dev #- upp-env@unified-dev #- ww3-env@unified-dev diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 812904a4f..36475e12a 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -8,11 +8,11 @@ Directory ``configs/sites`` contains site configurations for several HPC systems Pre-configured sites are split into two categories: Tier 1 with officially support spack-stack installations (see :numref:`Section %s `), and Tier 2 (sites with configuration files that were tested or contributed by others in the past, but that are not officially supported by the spack-stack team; see :numref:`Section %s `). .. _Preconfigured_Sites_Tier1: -======================================================= -Officially supported spack-stack installations (tier 1) -======================================================= +============================================================= +Officially supported spack-stack 1.3.0 installations (tier 1) +============================================================= -Ready-to-use spack-stack installations are available on the following, fully supported platforms. This version supports the JEDI Skylab release 4 of March 2023, and can be used for testing spack-stack with other applications (e.g. the UFS Weather Model, the UFS Short Range Weather Application, and the EMC Global Workflow). Amazon Web Services AMI are available on request in the US East 1 or 2 regions. +Ready-to-use spack-stack 1.3.0 installations are available on the following, fully supported platforms. This version can be used for testing spack-stack with UFS applications (e.g. the UFS Weather Model, the UFS Short Range Weather Application, and the EMC Global Workflow). Amazon Web Services AMI are available in the US East 1 or 2 regions. +------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ | System | Maintainers | Location | @@ -48,6 +48,30 @@ Ready-to-use spack-stack installations are available on the following, fully sup For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. +======================================================================= +Officially supported spack-stack 1.3.1 installations (subset of tier 1) +======================================================================= + +Ready-to-use spack-stack 1.3.1 installations are available on the following, fully supported platforms. This version supports the JEDI Skylab release 4 of March 2023, and can be used for testing spack-stack with other applications (e.g. the UFS Weather Model, the UFS Short Range Weather Application, and the EMC Global Workflow). Amazon Web Services AMI are available in the US East 1 or 2 regions. + ++------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ +| System | Maintainers | Location | ++============================================================+===============================+==============================================================================================================+ +| MSU Orion Intel/GNU | Cam Book / Dom Heinzeller | ``/work/noaa/epic-ps/role-epic-ps/spack-stack/spack-stack-1.3.1/envs/unified-env`` | ++------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ +| NASA Discover Intel/GNU | Dom Heinzeller / ??? | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.3.1/envs/unified-env`` | ++------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ +| NCAR-Wyoming Cheyenne Intel/GNU | Cam Book / Dom Heinzeller | ``/glade/work/epicufsrt/contrib/spack-stack/spack-stack-1.3.1/envs/unified-env`` | ++------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ +| UW (Univ. of Wisc.) S4 Intel/GNU | Dom Heinzeller / Mark Potts | ``/data/prod/jedi/spack-stack/spack-stack-1.3.1/envs/unified-env`` | ++------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ +| Amazon Web Services Parallelcluster Ubuntu 20.04 Intel/GNU | Dom Heinzeller / ??? | ``/mnt/experiments-efs/skylab-v4/spack-stack-1.3.1/envs/unified-env`` | ++------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ +| Amazon Web Services AMI Red Hat 8 GNU | Dom Heinzeller / ??? | ``/home/ec2-user/spack-stack/spack-stack-1.3.1/envs/unified-env`` | ++------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ + +For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. + .. _Preconfigured_Sites_Orion: ------------------------------ @@ -84,6 +108,26 @@ For ``spack-stack-1.3.0`` with GNU, load the following modules after loading min module load stack-python/3.9.7 module available +For ``spack-stack-1.3.1`` with Intel, load the following modules after loading miniconda and ecflow: + +.. code-block:: console + + module use /work/noaa/epic-ps/role-epic-ps/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core + module load stack-intel/2022.0.2 + module load stack-intel-oneapi-mpi/2021.5.1 + module load stack-python/3.9.7 + module available + +For ``spack-stack-1.3.1`` with GNU, load the following modules after loading miniconda and ecflow: + +.. code-block:: console + + module use /work/noaa/epic-ps/role-epic-ps/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core + module load stack-gcc/10.2.0 + module load stack-openmpi/4.0.4 + module load stack-python/3.9.7 + module available + .. _Preconfigured_Sites_Discover: ------------------------------ @@ -120,6 +164,26 @@ For ``spack-stack-1.3.0`` with GNU, load the following modules after loading min module load stack-python/3.9.7 module available +For ``spack-stack-1.3.1`` with Intel, load the following modules after loading miniconda and ecflow: + +.. code-block:: console + + module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core + module load stack-intel/2022.0.1 + module load stack-intel-oneapi-mpi/2021.5.0 + module load stack-python/3.9.7 + module available + +For ``spack-stack-1.3.1`` with GNU, load the following modules after loading miniconda and ecflow: + +.. code-block:: console + + module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core + module load stack-gcc/10.1.0 + module load stack-openmpi/4.1.3 + module load stack-python/3.9.7 + module available + .. _Preconfigured_Sites_Narwhal: ------------------------------ @@ -246,6 +310,26 @@ For ``spack-stack-1.3.0`` with GNU, load the following modules after loading min module load stack-python/3.9.12 module available +For ``spack-stack-1.3.1`` with Intel, load the following modules after loading miniconda and ecflow. + +.. code-block:: console + + module use /glade/work/epicufsrt/contrib/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core + module load stack-intel/19.1.1.217 + module load stack-intel-mpi/2019.7.217 + module load stack-python/3.9.12 + module available + +For ``spack-stack-1.3.1`` with GNU, load the following modules after loading miniconda and ecflow: + +.. code-block:: console + + module use /glade/work/epicufsrt/contrib/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core + module load stack-gcc/10.1.0 + module load stack-openmpi/4.1.1 + module load stack-python/3.9.12 + module available + .. _Preconfigured_Sites_Acorn: ------------------------------- @@ -429,7 +513,30 @@ For ``spack-stack-1.3.0`` with GNU, load the following modules after loading min .. code-block:: console module use /data/prod/jedi/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core + module load stack-gcc/9.3.0 + module load stack-mpich/4.0.1 + module load stack-python/3.9.12 + module available + +For ``spack-stack-1.3.1`` with Intel, load the following modules after loading miniconda and ecflow: + +.. code-block:: console + + module use /data/prod/jedi/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core + module load stack-intel/2021.5.0 + module load stack-intel-oneapi-mpi/2021.5.0 + module load stack-python/3.9.12 + module unuse /opt/apps/modulefiles/Compiler/intel/non-default/22 + module unuse /opt/apps/modulefiles/Compiler/intel/22 + module available + +Note the two `module unuse` commands, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. + +For ``spack-stack-1.3.1`` with GNU, load the following modules after loading miniconda and ecflow: + +.. code-block:: console + module use /data/prod/jedi/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core module load stack-gcc/9.3.0 module load stack-mpich/4.0.1 module load stack-python/3.9.12 @@ -454,7 +561,7 @@ For ``spack-stack-1.3.0`` with Intel, run the following commands/load the follow module load stack-python/3.10.8 module available -For ``spack-stack-1.3.0`` with Intel, run the following commands/load the following modules: +For ``spack-stack-1.3.0`` with GNU, run the following commands/load the following modules: module purge ulimit -s unlimited @@ -464,11 +571,36 @@ For ``spack-stack-1.3.0`` with Intel, run the following commands/load the follow module load stack-python/3.10.8 module available +For ``spack-stack-1.3.1`` with Intel, run the following commands/load the following modules: + +.. code-block:: console + + module purge + ulimit -s unlimited + source /opt/intel/oneapi/compiler/2022.1.0/env/vars.sh + module use /mnt/experiments-efs/skylab-v4/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core + module load stack-intel/2022.1.0 + module load stack-intel-oneapi-mpi/2021.6.0 + module load stack-python/3.10.8 + module available + +For ``spack-stack-1.3.1`` with GNU, run the following commands/load the following modules: + + module purge + ulimit -s unlimited + module use /mnt/experiments-efs/skylab-v4/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core + module load stack-gcc/9.4.0 + module load stack-openmpi/4.1.4 + module load stack-python/3.10.8 + module available + ----------------------------- Amazon Web Services Red Hat 8 ----------------------------- -For ``spack-stack-1.3.0`` use a c6i.4xlarge instance or similar with AMI "skylab-4.0.0-redhat8" (ami-098a3fdd801055c14 in region us-east-1, ami-039759644cac741eb in region us-east-2). After logging in, run: +Use a c6i.4xlarge instance or similar with AMI "skylab-4.0.0-redhat8-update-20230419" (ami-01e025b0334795ffa in region us-east-1, TODO MAKE PUBLIC ami-094361e081de18589 in region us-east-2). + +For ``spack-stack-1.3.0``, run: .. code-block:: console @@ -480,6 +612,18 @@ For ``spack-stack-1.3.0`` use a c6i.4xlarge instance or similar with AMI "skylab module load stack-python/3.10.8 module available +For ``spack-stack-1.3.1``, run: + +.. code-block:: console + + ulimit -s unlimited + scl enable gcc-toolset-11 bash + module use /home/ec2-user/spack-stack/spack-stack-1.3.1/envs/unified-env/install/modulefiles/Core + module load stack-gcc/11.2.1 + module load stack-openmpi/4.1.4 + module load stack-python/3.10.8 + module available + .. _Preconfigured_Sites_Tier2: ============================= Pre-configured sites (tier 2) diff --git a/spack b/spack index 4df1a0cc8..a0b424718 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 4df1a0cc83b34dadc177fb1011d9624e97aa89b4 +Subproject commit a0b424718417ef8693498e61071d5062d9343a4f