Merge pull request #373 from xylar/add_matrix_build

Add utility for matrix builds

conda/bootstrap.py

@@ -746,6 +746,13 @@ def main():
     compilers, mpis = get_compilers_mpis(config, machine, args.compilers,
                                          args.mpis, source_path)
 
+    if machine is not None:
+        # write out a log file for use by matrix builds
+        with open('conda/logs/matrix.log', 'w') as f:
+            f.write(f'{machine}\n')
+            for compiler, mpi in zip(compilers, mpis):
+                f.write(f'{compiler}, {mpi}\n')
+
     print('Configuring environment(s) for the following compilers and MPI '
           'libraries:')
     for compiler, mpi in zip(compilers, mpis):

utils/matrix/README.md

@@ -0,0 +1,98 @@
+Matrix build and setup
+======================
+
+Often, developers want to test a given set of test cases or a test suite with
+multiple compilers and configurations.  Doing this process manually can be 
+challenging and prone to mistakes.  The `setup_matrix.py` scrip is designed
+to automate this process with the help of a config file similar to 
+`example.cfg`.
+
+Instructions
+------------
+
+1. Configure the compass environment and create load scripts with a matrix of
+   compilers and mpi libraries as desired, e.g.:
+   ```shell
+   ./conda/configure_compass_env.py --env_name compass_matrix \
+       --compiler all --mpi all --conda ~/miniconda3/
+   ```
+   See the next section for more details.  This will save some info in 
+   `conda/logs/matrix.log` that is needed for the remainder of the build.  If
+   you want to set up load scripts for more environments than you actually
+   want to run your matrix on, you can edit `matrix.log`.  The syntax is:
+   ```
+   <machine>
+   <compiler>, <mpi>
+   <compiler>, <mpi>
+   <compiler>, <mpi>
+   ...
+   ```
+   It is also safe to rerun `./conda/configure_compass_env.py` with just the
+   subset of compilers and MPI libraries you want for the matrix.  Any
+   load scripts created in previous calls will not be deleted and the conda
+   environment will just be updated, not recreated unless you explicitly ask
+   for it to be recreated.
+
+2. Copy `example.cfg` to the base of the branch:
+   ```shell
+   cp utils/matrix/example.cfg matrix.cfg
+   ```
+
+3. Modify the config options with the appropriate compilers and MPI libraries;
+   whether to build in debug mode, optimized or both; and whether to build with
+   OpenMPI or not (or both)
+
+4. Set the conda environment name (or prefix on Linux and OSX) that you want to
+   use
+
+5. Modify the various paths and commands as needed.
+
+6. Add any other config options you want to pass on to the `compass setup` or
+   `compass suite` command.  A config file will be written out for each
+   build configuration you select.
+
+7. On a login node, run:
+   ```shell
+   ./utils/matrix/setup_matrix.py -f matrix.cfg
+   ```
+   Optionally use the `--submit` flag to submit jobs once each configuration
+   has been built and set up.
+
+8. The matrix build doesn't take care of running the jobs on a compute node.
+   You will need to do that yourself.
+
+Matrix of compilers and MPI libraries
+-------------------------------------
+
+You control the matrix of compilers and MPI libraries by how you call
+```shell
+./conda/configure_compass_env.py ...
+```
+This is because the configure script knows about which compilers and MPI 
+libraries are  available for a given machine, something tricky to figure out in 
+`./util/matrix/setup_matrix.py` directly.
+
+You can do:
+```shell
+./conda/configure_compass_env.py --compiler all --mpi all ...
+```
+to get all supported compilers and MPI libraries.  You can do:
+```shell
+./conda/configure_compass_env.py --compiler intel --mpi all ...
+```
+to get just the `intel` configurations.  You can do:
+```shell
+./conda/configure_compass_env.py --compiler all --mpi openmpi ...
+```
+to get just the `openmpi` configurations or:
+```shell
+./conda/configure_compass_env.py --compiler intel ...
+```
+(omitting the `--mpi`) to get the default MPI library for intel.  Finally, you
+can give a list of compilers and a list of the same length of MPI libraries,
+such as:
+```shell
+./conda/configure_compass_env.py --compiler intel intel intel gnu \
+    --mpi impi openmpi mvapich mvapich ...
+```
+This will give you 3 intel variants and a gnu variant.

utils/matrix/example.cfg

@@ -0,0 +1,37 @@
+# config options related to building MPAS and set up compass with a matrix of
+# build configurations
+[matrix]
+
+# OpenMP options: "True, False", "True" or "False"
+openmp = True
+
+# Debug options: "True, False", "True" or "False"
+debug = True, False
+
+# The environment name (or prefix on Linux or OSX)
+env_name = compass_test
+
+# Additional flags for the build command, for example:
+# other_build_flags = GEN_F90=true
+other_build_flags =
+
+# the absolute or relative path to the MPAS model directory you want to build
+mpas_path = E3SM-Project/components/mpas-ocean
+
+# the absolute or relative path for test results (subdirectories will be
+# created within this path for each build configuration)
+work_base = /lcrc/group/e3sm/ac.xylar/compass_1.0/anvil/test_20220417/matrix
+
+# the absolute or relative path for a baseline for comparison already build
+# and run with the same build matrix, for example:
+# baseline_base = /lcrc/group/e3sm/ac.xylar/compass_1.0/anvil/test_20220417/matrix_baseline
+# The default is no baseline
+baseline_base =
+
+# the command to set up one or more test cases or a test suite
+# note: the mpas model, work directory and baseline directory will be appended
+#       automatically so don't include -p, -w or -b flags
+setup_command = compass suite -s -c ocean -t nightly
+
+
+# Include other config sections and options you want to pass on to compass here