Merge release/1.5.0 into develop

.github/workflows/macos-ci-aarch64.yaml

@@ -111,6 +111,12 @@ jobs:
           echo "Packages in combined spack build caches:"
           spack buildcache list
 
+          # Workaround for limited disk space on macOS arm instance
+          spack config add "config:build_stage:/Users/ec2-user/spack-stack/spack-cache/build_stage"
+          spack config add "config:test_stage:/Users/ec2-user/spack-stack/spack-cache/test_stage"
+          spack config add "config:source_cache:/Users/ec2-user/spack-stack/spack-cache/source_cache"
+          spack config add "config:misc_cache:/Users/ec2-user/spack-stack/spack-cache/misc_cache"
+
           # Break installation up in pieces and create build caches in between
           # This allows us to "spin up" builds that altogether take longer than
           # six hours, and/or fail later in the build process.

configs/common/packages.yaml

@@ -11,13 +11,13 @@
         lapack: [openblas]
         yacc: [bison]
     #
+    # This version of awscli goes with py-pyyaml@5.4.1
     awscli:
-      version: ['1.29.41']
-    bacio:  
+      version: ['1.27.84']
+    bacio:
       version: ['2.4.1']
     bison:
       version: ['3.8.2']
-    # Attention - when updating also check orion site config
     boost:
       version: ['1.78.0']
       variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden
@@ -103,7 +103,7 @@
       # the container builds.
       #version: ['2.11.0']
     grib-util:
-      version: ['1.2.3']
+      version: ['1.3.0']
     gsibec:
       version: ['1.1.3']
     gsi-ncdiag:
@@ -117,7 +117,7 @@
       version: ['1.14.0']
       variants: +hl +fortran +mpi ~threadsafe +szip
     ip:
-      version: ['3.3.3']
+      version: ['4.3.0']
     ip2:
       version: ['1.1.2']
     jasper:
@@ -137,8 +137,12 @@
       version: ['0.2.5']
     mapl:
       # 2.35.2 goes with esmf@8.4.2, 2.40.3 goes with esmf@8.5.0
+      # turn off ~pflogger and extdata2g to avoid compilation
+      # errors with intel@2021.7.0+, see
+      # https://github.com/JCSDA/spack-stack/issues/769
+      # also: ... extdata2g segfault UFS?
       version: ['2.35.2']
-      variants: ~shared
+      variants: ~shared ~extdata2g ~pflogger
     # If making changes here, also check the Discover site config and the CI workflows
     met:
       version: ['11.1.0']
@@ -260,8 +264,9 @@
       # Versions earlier than 0.11.0 don't compile on macOS with llvm-clang/13.0.0 and Python/3.9,
       # and 0.11.0 leads to downstream errors in py-scipy with the Intel compilers
       version: ['0.12.2']
+    # This version of py-pyyaml goes with awscli@1.27.84
     py-pyyaml:
-      version: ['6.0']
+      version: ['5.4.1']
     py-scipy:
       version: ['1.9.3']
     # Pin the py-setuptools version to avoid duplicate Python packages
@@ -298,7 +303,7 @@
     upp:
       version: ['10.0.10']
     w3emc:
-      version: ['2.9.2']
+      version: ['2.10.0']
     w3nco:
       version: ['2.4.1']
     wget:

configs/containers/README.md

@@ -8,38 +8,21 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li
     jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@12.0.0, ecbuild@3.7.2, eccodes@2.27.0, ecflow@5,
     eckit@1.24.4, ecmwf-atlas@0.34.0 +trans ~fftw, fiat@1.2.0, ectrans@1.2.0 ~fftw, eigen@3.4.0,
     fckit@0.11.0, fms@release-jcsda, g2@3.4.5, g2tmpl@1.10.0, gftl-shared@1.5.0,
-    gsibec@1.1.3, hdf@4.2.15, hdf5@1.14.1-2, ip@3.3.3, jasper@2.0.32, jedi-cmake@1.4.0,
+    gsibec@1.1.3, hdf@4.2.15, hdf5@1.14.0, ip@4.3.0, jasper@2.0.32, jedi-cmake@1.4.0,
     libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.9.2, netcdf-cxx4@4.3.1,
     netcdf-fortran@4.6.0, nlohmann-json@3.10.5, nlohmann-json-schema-validator@2.1.0,
     parallelio@2.5.10, parallel-netcdf@1.12.2, py-eccodes@1.4.2, py-f90nml@1.4.3,
     py-gitpython@3.1.27, py-h5py@3.7.0, py-numpy@1.22.3,
-    py-pandas@1.4.0, py-pip, py-pyyaml@6.0, py-scipy@1.9.3, py-shapely@1.8.0, py-xarray@2022.3.0,
-    sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, nco@5.0.6, esmf@8.4.2, mapl@2.35.2,
-    yafyaml@0.5.1, zlib@1.2.13, odc@1.4.6, shumlib@macos_clang_linux_intel_port]
-    # Don't build CRTM by default so that it gets built in the JEDI bundles:
-    # crtm@v2.4.1-jedi
-    # Comment out for now until build problems are solved
-    # https://github.com/jcsda/spack-stack/issues/522
-    # py-mysql-connector-python@8.0.32
-```
-
-### spack-stack-1.5.0 / ufs-weather-model-x.y.z containers for ufs-weather-model as of July 5, 2023
-
-**Note. This is not yet working correctly, some libraries are missing. Please do not use yet! Also, if using the clang-mpich container, need to disable openmp for fms, not clear how to do this cleanly.**
-
-```
-  specs: [base-env@1.0.0,
-    bacio@2.4.1, bison@3.8.2, bufr@12.0.0, ecbuild@3.7.2, eccodes@2.27.0, ecflow@5,
-    eckit@1.24.4, ecmwf-atlas@0.34.0 +trans ~fftw, fiat@1.2.0, ectrans@1.2.0 ~fftw, eigen@3.4.0,
-    fckit@0.11.0, fms@2023.01, g2@3.4.5, g2tmpl@1.10.0, gftl-shared@1.5.0,
-    gsibec@1.1.3, hdf@4.2.15, hdf5@1.14.1-2, ip@3.3.3, jasper@2.0.32, jedi-cmake@1.4.0,
-    libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.9.2, netcdf-cxx4@4.3.1,
-    netcdf-fortran@4.6.0, nlohmann-json@3.10.5, nlohmann-json-schema-validator@2.1.0,
-    parallelio@2.5.10, parallel-netcdf@1.12.2, py-eccodes@1.4.2, py-f90nml@1.4.3,
-    py-gitpython@3.1.27, py-h5py@3.7.0, py-numpy@1.22.3, py-pandas@1.4.0,
-    py-pip, py-pyyaml@6.0, py-scipy@1.9.3, py-shapely@1.8.0, py-xarray@2022.3.0,
-    sp@2.3.3, udunits@2.2.28, w3emc@2.9.2, w3nco@2.4.1, nco@5.0.6, esmf@8.4.2, mapl@2.35.2,
-    yafyaml@0.5.1, zlib@1.2.13, odc@1.4.6, crtm@2.4.0, crtm-fix@2.4.0_emc]
+    py-pandas@1.5.3, py-pip, py-pyyaml@5.4.1, py-scipy@1.9.3, py-shapely@1.8.0, py-xarray@2022.3.0,
+    sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, w3emc@2.10.0, nco@5.0.6, esmf@8.4.2, mapl@2.35.2,
+    yafyaml@0.5.1, zlib@1.2.13, zstd@1.5.2, odc@1.4.6, shumlib@macos_clang_linux_intel_port,
+    awscli@1.27.84, py-globus-cli@3.16.0]
+    # Notes:
+    # 1. Remove mapl@2.35.2 from clang/mpich container, because mapl doesn't work with mpich@4
+    # 2. Don't build CRTM by default so that it gets built in the JEDI bundles
+    # 3. Comment out for now until build problems are solved
+    #    https://github.com/jcsda/spack-stack/issues/522
+    #    py-mysql-connector-python@8.0.32
 ```
 
 ### Create an AMI on AWS EC2 to build docker containers

configs/containers/docker-ubuntu-clang-mpich.yaml

@@ -38,6 +38,11 @@ spack:
       externals:
       - spec: gcc@9.4.0
         prefix: /usr
+    gmake:
+      buildable: false
+      externals:
+      - spec: gmake@4.2.1
+        prefix: /usr
     diffutils:
       buildable: false
       externals:
@@ -105,7 +110,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/jcsda/spack
-        ref: jcsda_emc_spack_stack
+        ref: spack-stack-1.5.0
         resolve_sha: false
 
     # Whether or not to strip binaries
@@ -235,8 +240,6 @@ spack:
         echo "export CXX=clang++" >> /root/.bashrc && \
         echo "export FC=gfortran" >> /root/.bashrc && \
         printf "[credential]\n    helper = cache --timeout=7200\n" >> /root/.gitconfig
-        # Source spack environment script for root
-        RUN echo "source /etc/profile.d/z10_spack_environment.sh\n" >> /root/.bashrc
 
     # Labels for the image
     labels:

configs/containers/docker-ubuntu-gcc-openmpi.yaml

@@ -36,6 +36,12 @@ spack:
       externals:
       - spec: gcc@9.4.0
         prefix: /usr
+    # Try this next time, already used for clang/intel
+    #gmake:
+    #  buildable: false
+    #  externals:
+    #  - spec: gmake@4.2.1
+    #    prefix: /usr
     diffutils:
       buildable: false
       externals:
@@ -92,7 +98,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/jcsda/spack
-        ref: jcsda_emc_spack_stack
+        ref: spack-stack-1.5.0
         resolve_sha: false
 
     # Whether or not to strip binaries
@@ -184,8 +190,6 @@ spack:
         RUN echo "export OMPI_ALLOW_RUN_AS_ROOT=1" >> /root/.bashrc && \
         echo "export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >> /root/.bashrc && \
         echo "export OMPI_MCA_rmaps_base_oversubscribe=1" >> /root/.bashrc
-        # Source spack environment script for root
-        RUN echo "source /etc/profile.d/z10_spack_environment.sh\n" >> /root/.bashrc
 
     # Labels for the image
     labels:

configs/containers/docker-ubuntu-intel-impi.yaml

@@ -50,6 +50,11 @@ spack:
       externals:
       - spec: intel-oneapi-mkl@2022.1.0
         prefix: /opt/intel/oneapi
+    gmake:
+      buildable: false
+      externals:
+      - spec: gmake@4.2.1
+        prefix: /usr
     diffutils:
       buildable: false
       externals:
@@ -110,7 +115,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/jcsda/spack
-        ref: jcsda_emc_spack_stack
+        ref: spack-stack-1.5.0
         resolve_sha: false
 
     # Whether or not to strip binaries
@@ -211,8 +216,6 @@ spack:
         echo "source /opt/intel/oneapi/compiler/latest/env/vars.sh" >> /root/.bashrc && \
         echo "source /opt/intel/oneapi/mpi/latest/env/vars.sh" >> /root/.bashrc && \
         printf "[credential]\n    helper = cache --timeout=7200\n" >> /root/.gitconfig
-        # Source spack environment script for root
-        RUN echo "source /etc/profile.d/z10_spack_environment.sh\n" >> /root/.bashrc
 
     # Labels for the image
     labels:

configs/sites/acorn/compilers.yaml

@@ -20,6 +20,27 @@ compilers:
         # which confuses some packages.
         CONFIG_SITE: ''
     extra_rpaths: []
+- compiler:
+    spec: intel@2022.0.2.262
+    paths:
+      cc: cc
+      cxx: CC
+      f77: ftn
+      fc: ftn
+    flags: {}
+    operating_system: sles15
+    modules:
+    - PrgEnv-intel/8.3.3
+    - craype/2.7.13
+    - intel-classic/2022.2.0.262
+    - libfabric
+    environment:
+      set:
+        # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so
+        # Automake-based builds are installed in lib64
+        # which confuses some packages.
+        CONFIG_SITE: ''
+    extra_rpaths: []
 - compiler:
     spec: gcc@11.2.0
     paths:

configs/sites/acorn/modules.yaml

@@ -2,3 +2,6 @@ modules:
   default:
     enable::
     - lmod
+    lmod:
+      include:
+      - python

configs/sites/acorn/packages.yaml

@@ -1,6 +1,6 @@
   packages:
     all:
-      compiler:: [intel@19.1.3.304,gcc@11.2.0]
+      compiler:: [intel@19.1.3.304,intel@2022.0.2.262,gcc@11.2.0]
       providers:
         mpi:: [cray-mpich@8.1.9]
     cray-mpich:
@@ -22,13 +22,6 @@
       externals:
       - spec: git-lfs@2.11.0
         modules: [git-lfs/2.11.0]
-    python:
-      buildable: false
-      externals:
-      - spec: python@3.8.6%intel
-        prefix: /apps/spack/python/3.8.6/intel/19.1.3.304/pjn2nzkjvqgmjw4hmyz43v5x4jbxjzpk
-      - spec: python@3.8.6%gcc
-        prefix: /apps/spack/python/3.8.6/gcc/11.2.0/gr4zqejk5oxwsrmxvkenmye4nni4zt6e
     perl:
       buildable: false
       externals:
@@ -61,3 +54,7 @@
       externals:
       - spec: flex@2.6.4+lex
         prefix: /usr
+    eckit:
+      require:
+      - any_of: ["@1.23.1"]
+        when: "%intel@19.1.3.304"

configs/sites/aws-pcluster/modules.yaml

@@ -4,7 +4,6 @@ modules:
     - lmod
     lmod:
       include:
-      # List of packages for which we need modules that are blacklisted by default
       - python
       exclude:
       - ecflow

configs/sites/aws-pcluster/packages.yaml

@@ -43,21 +43,11 @@ packages:
     externals:
     - spec: binutils@2.34
       prefix: /usr
-  bison:
-    externals:
-    - spec: bison@3.5.1
-      prefix: /usr
   boost:
     buildable: False
     externals:
     - spec: boost@1.71.0 +atomic +chrono +date_time +exception +filesystem +graph +iostreams +locale +log +math +mpi +numpy +pic +program_options +python +random +regex +serialization +signals +system +test +thread +timer +wave cxxstd=17 visibility=hidden
       prefix: /usr
-  # Don't use external cmake, this confuses the spack concretizer
-  # leading to duplicate packages being installed.
-  #cmake:
-  #  externals:
-  #  - spec: cmake@3.16.3
-  #    prefix: /usr
   coreutils:
     externals:
     - spec: coreutils@8.30
@@ -120,13 +110,6 @@ packages:
     externals:
     - spec: openssh@8.2p1
       prefix: /usr
-  # Don't use external openssl, this confuses the spack concretizer
-  # leading to duplicate packages (libarchive with different compression
-  # settings, somehow triggered by different cmake versions) being installed.
-  #openssl:
-  #  externals:
-  #  - spec: openssl@1.1.1f
-  #    prefix: /usr
   perl:
     externals:
     - spec: perl@5.30.0~cpanm+shared+threads

configs/sites/casper/compilers.yaml

@@ -1,23 +1,24 @@
 compilers::
   - compiler:
-      spec: intel@19.1.1.217
+      spec: intel@2022.0.2
       paths:
-        cc: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/icc
-        cxx: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/icpc
-        f77: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/ifort
-        fc: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/ifort
+        cc: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/icc
+        cxx: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/icpc
+        f77: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/ifort
+        fc: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/ifort
       flags: {}
       operating_system: centos7
       target: x86_64
       modules:
-      - intel/19.1.1
+      - intel/2022.1
       environment:
         prepend_path:
-          PATH: '/glade/u/apps/dav/opt/gnu/9.1.0/bin'
-          CPATH: '/glade/u/apps/dav/opt/gnu/9.1.0/include'
-          LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/lib/intel64_lin:/glade/u/apps/opt/intel/2020u1/compilers_and_libraries_2020.1.217/linux/mpi/intel64/libfabric/lib:/glade/u/apps/dav/opt/gnu/9.1.0/lib64'
+          PATH: '/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/bin'
+          CPATH: '/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/include'
+          LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2022.1/compiler/latest/linux/compiler/lib/intel64_lin:/glade/u/apps/opt/intel/2022.1/mpi/2021.5.1/libfabric/lib:/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/lib64'
         set:
           INTEL_LICENSE_FILE: '28518@128.117.177.41'
           LM_LICENSE_FILE: '28518@128.117.177.41'
-          I_MPI_ROOT: '/glade/u/apps/opt/intel/2020u1/impi/2019.7.217/intel64'
+          I_MPI_ROOT: '/glade/u/apps/opt/intel/2022.1/mpi/2021.5.1'
+
       extra_rpaths: []

configs/sites/casper/modules.yaml

@@ -5,3 +5,5 @@ modules:
     lmod:
       exclude:
       - ecflow
+      include:
+      - python