update to dev/emc

jiandewang · Jul 1, 2021 · 202cbd4 · 202cbd4
2 parents 61717ee + b418ae1
commit 202cbd4
Show file tree

Hide file tree

Showing 337 changed files with 34,659 additions and 14,036 deletions.
diff --git a/.github/actions/testing-setup/action.yml b/.github/actions/testing-setup/action.yml
@@ -0,0 +1,77 @@
+name: 'Build-.testing-prerequisites'
+description: 'Build pre-requisites for .testing including FMS and a symmetric MOM6 executable'
+inputs:
+  build_symmetric:
+    description: 'If true, will build the symmetric MOM6 executable'
+    required: false
+    default: 'true'
+  install_python:
+    description: 'If true, will install the local python env needed for .testing'
+    required: false
+    default: 'true'
+runs:
+  using: 'composite'
+  steps:
+    - name: Git info
+      shell: bash
+      run: |
+        echo "::group::Git commit info"
+        echo "git log:"
+        git log | head -60
+        echo "::endgroup::"
+
+    - name: Env
+      shell: bash
+      run: |
+        echo "::group::Environment"
+        env
+        echo "::endgroup::"
+
+    - name: Install needed packages for compiling
+      shell: bash
+      run: |
+        echo "::group::Install linux packages"
+        sudo apt-get update
+        sudo apt-get install netcdf-bin libnetcdf-dev libnetcdff-dev mpich libmpich-dev
+        echo "::endgroup::"
+
+    - name: Compile FMS library
+      shell: bash
+      run: |
+        echo "::group::Compile FMS library"
+        cd .testing
+        make deps/lib/libFMS.a -s -j
+        echo "::endgroup::"
+
+    - name: Store compiler flags used in Makefile
+      shell: bash
+      run: |
+        echo "::group::config.mk"
+        cd .testing
+        echo "FCFLAGS_DEBUG=-g -O0 -Wextra -Wno-compare-reals -fbacktrace -ffpe-trap=invalid,zero,overflow -fcheck=bounds" >> config.mk
+        echo "FCFLAGS_REPRO=-g -O2 -fbacktrace" >> config.mk
+        echo "FCFLAGS_INIT=-finit-real=snan -finit-integer=2147483647 -finit-derived" >> config.mk
+        echo "FCFLAGS_COVERAGE=--coverage" >> config.mk
+        cat config.mk
+        echo "::endgroup::"
+
+    - name: Compile MOM6 in symmetric memory mode
+      shell: bash
+      run: |
+        echo "::group::Compile MOM6 in symmetric memory mode"
+        cd .testing
+        test ${{ inputs.build_symmetric }} == true && make build/symmetric/MOM6 -j
+        echo "::endgroup::"
+
+    - name: Install local python venv for generating input data
+      shell: bash
+      run: |
+        echo "::group::Create local python env for input data generation"
+        cd .testing
+        test ${{ inputs.install_python }} == true && make work/local-env
+        echo "::endgroup::"
+
+    - name: Set flags
+      shell: bash
+      run: |
+        echo "TIMEFORMAT=... completed in %lR  (user: %lU, sys: %lS)" >> $GITHUB_ENV
diff --git a/.github/workflows/coupled-api.yml b/.github/workflows/coupled-api.yml
@@ -0,0 +1,33 @@
+name: API for coupled drivers
+
+on: [push, pull_request]
+
+jobs:
+  test-top-api:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+      with:
+        build_symmetric: 'false'
+        install_python: 'false'
+
+    - name: Compile MOM6 for the GFDL coupled driver
+      shell: bash
+      run: make check_mom6_api_coupled -j
+
+    - name: Compile MOM6 for the NUOPC driver
+      shell: bash
+      run: make check_mom6_api_nuopc -j
+
+    - name: Compile MOM6 for the MCT driver
+      shell: bash
+      run: make check_mom6_api_mct -j
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -0,0 +1,24 @@
+name: Code coverage
+
+on: [push, pull_request]
+
+jobs:
+  build-test-nans:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    env:
+      REPORT_COVERAGE: true
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Run and post coverage
+      run: make run.symmetric -k -s
diff --git a/.github/workflows/documentation-and-style.yml b/.github/workflows/documentation-and-style.yml
@@ -0,0 +1,39 @@
+name: Doxygen and style
+
+on: [push, pull_request]
+
+jobs:
+  doxygen:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Check white space (non-blocking)
+      run: |
+        ./.testing/trailer.py -e TEOS10 -l 120 src config_src 2>&1 | tee style_errors
+      continue-on-error: true
+
+    - name: Install packages used when generating documentation
+      run: |
+        sudo apt-get update
+        sudo apt-get install python3-sphinx python3-lxml perl
+        sudo apt-get install texlive-binaries texlive-base bibtool tex-common texlive-bibtex-extra
+        sudo apt-get install graphviz
+
+    - name: Build doxygen HTML
+      run: |
+        cd docs
+        perl -e 'print "perl version $^V" . "\n"'
+        mkdir _build && make nortd DOXYGEN_RELEASE=Release_1_8_13 UPDATEHTMLEQS=Y
+        cat _build/doxygen_warn_nortd_log.txt
+
+    - name: Report doxygen or style errors
+      run: |
+        grep "warning:" docs/_build/doxygen_warn_nortd_log.txt | grep -v "as part of a" | tee doxy_errors
+        cat style_errors doxy_errors > all_errors
+        cat all_errors
+        test ! -s  all_errors
diff --git a/.github/workflows/expression.yml b/.github/workflows/expression.yml
@@ -0,0 +1,27 @@
+name: Expression verification
+
+on: [push, pull_request]
+
+jobs:
+  test-repro-and-dims:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Compile MOM6 using repro optimization
+      run: make build/repro/MOM6 -j
+
+    - name: Create validation data
+      run: make run.symmetric -k -s
+
+    - name: Run tests
+      run: make test.repros test.dims -k -s
diff --git a/.github/workflows/other.yml b/.github/workflows/other.yml
@@ -0,0 +1,27 @@
+name: OpenMP and Restart verification
+
+on: [push, pull_request]
+
+jobs:
+  test-openmp-nan-restarts:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Compile with openMP
+      run: make build/openmp/MOM6 -j
+
+    - name: Create validation data
+      run: make run.symmetric -k -s
+
+    - name: Run tests
+      run: make test.openmps test.nans test.restarts -k -s
diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
@@ -0,0 +1,27 @@
+name: Regression
+
+on: [pull_request]
+
+jobs:
+  build-test-regression:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Compile reference model
+      run: make build.regressions MOM_TARGET_SLUG=$GITHUB_REPOSITORY MOM_TARGET_LOCAL_BRANCH=$GITHUB_BASE_REF DO_REGRESSION_TESTS=true -j
+
+    - name: Create validation data
+      run: make run.symmetric -k -s
+
+    - name: Regression test
+      run: make test.regressions DO_REGRESSION_TESTS=true -k -s
diff --git a/.github/workflows/stencil.yml b/.github/workflows/stencil.yml
@@ -0,0 +1,27 @@
+name: Stencil related verification
+
+on: [push, pull_request]
+
+jobs:
+  test-symmetric-layout-rotation:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Compile MOM6 in asymmetric memory mode
+      run: make build/asymmetric/MOM6 -j
+
+    - name: Create validation data
+      run: make run.symmetric -k -s
+
+    - name: Run tests
+      run: make test.grids test.layouts test.rotations -k -s
diff --git a/.gitignore b/.gitignore
@@ -4,13 +4,7 @@
 html
 
 
-# Build output
-*.o
-*.mod
-MOM6
-
-
-# Autoconf
+# Autoconf output
 aclocal.m4
 autom4te.cache/
 config.log

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -31,7 +31,7 @@ setup:
     # Clone regressions directory
     - git clone --recursive http://gitlab.gfdl.noaa.gov/ogrp/Gaea-stats-MOM6-examples.git tests && cd tests
     # Install / update testing scripts
-    - git clone https://github.com/adcroft/MRS.git MRS
+    - git clone -b new-code-struct https://github.com/adcroft/MRS.git MRS
     # Update MOM6-examples and submodules
     - (cd MOM6-examples && git checkout . && git checkout dev/gfdl && git pull && git submodule init && git submodule update)
     - (cd MOM6-examples/src/MOM6 && git submodule update)
@@ -60,7 +60,7 @@ gnu:ocean-only-nolibs:
     - time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
     - make -f MRS/Makefile.build build/gnu/env && cd build/gnu
     # mkdir -p build/gnu/repro/symmetric_dynamic/ocean_only && cd build/gnu/repro/symmetric_dynamic/ocean_only
-    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{solo_driver,dynamic_symmetric,ext*} ../../../src ../../MOM6-examples/src/FMS
+    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{drivers/solo_driver,memory/dynamic_symmetric,infra/FMS1,ext*} ../../../src ../../MOM6-examples/src/FMS
     - sed -i '/FMS\/.*\/test_/d' path_names
     - ../../MOM6-examples/src/mkmf/bin/mkmf -t ../../MOM6-examples/src/mkmf/templates/ncrc-gnu.mk -p MOM6 -c"-Duse_libMPI -Duse_netCDF" path_names
     - time (source ./env ; make NETCDF=3 REPRO=1 MOM6 -s -j)
@@ -73,7 +73,7 @@ gnu:ice-ocean-nolibs:
     - time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
     - make -f MRS/Makefile.build build/gnu/env && cd build/gnu
     # mkdir -p build/gnu/repro/symmetric_dynamic/ocean_only && cd build/gnu/repro/symmetric_dynamic/ocean_only
-    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{coupled_driver,dynamic,ext*} ../../../src ../../MOM6-examples/src/{FMS,coupler,SIS2,icebergs,ice_param,land_null,atmos_null}
+    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{drivers/FMS_cap,memory/dynamic_nonsymmetric,infra/FMS1,ext*} ../../../src ../../MOM6-examples/src/{FMS,coupler,SIS2,icebergs,ice_param,land_null,atmos_null}
     - sed -i '/FMS\/.*\/test_/d' path_names
     - ../../MOM6-examples/src/mkmf/bin/mkmf -t ../../MOM6-examples/src/mkmf/templates/ncrc-gnu.mk -p MOM6 -c"-Duse_libMPI -Duse_netCDF -D_USE_LEGACY_LAND_ -Duse_AM3_physics" path_names
     - time (source ./env ; make NETCDF=3 REPRO=1 MOM6 -s -j)
@@ -117,11 +117,36 @@ run:
     - time tar zxf $CACHE_DIR/build-pgi-repro-$CI_PIPELINE_ID.tgz
     # time tar zxf $CACHE_DIR/build-gnu-debug-$CI_PIPELINE_ID.tgz
     - (echo '#!/bin/tcsh';echo 'make -f MRS/Makefile.tests all') > job.sh
-    - sbatch --clusters=c3,c4 --nodes=29 --time=0:34:00 --account=gfdl_o --qos=debug --job-name=mom6_regressions --output=log.$CI_PIPELINE_ID --wait job.sh
+    - sbatch --clusters=c3,c4 --nodes=29 --time=0:34:00 --account=gfdl_o --qos=debug --job-name=mom6_regressions --output=log.$CI_PIPELINE_ID --wait job.sh || MJOB_RETURN_STATE=Fail
     - cat log.$CI_PIPELINE_ID
+    - test -z "$MJOB_RETURN_STATE"
     - test -f restart_results_gnu.tar.gz
     - time tar zvcf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz *.tar.gz
 
+gnu.testing:
+  stage: run
+  tags:
+    - ncrc4
+  script:
+    - cd .testing
+    - module unload PrgEnv-pgi PrgEnv-intel PrgEnv-gnu darshan ; module load PrgEnv-gnu ; module unload netcdf gcc ; module load gcc/7.3.0 cray-hdf5 cray-netcdf
+    - make work/local-env
+    - make -s -j
+    - (echo '#!/bin/bash';echo '. ./work/local-env/bin/activate';echo 'make MPIRUN="srun -mblock --exclusive" test -s -j') > job.sh
+    - sbatch --clusters=c3,c4 --nodes=5 --time=0:05:00 --account=gfdl_o --qos=debug --job-name=MOM6.gnu.testing --output=log.$CI_PIPELINE_ID --wait job.sh || cat log.$CI_PIPELINE_ID && make test
+
+intel.testing:
+  stage: run
+  tags:
+    - ncrc4
+  script:
+    - cd .testing
+    - module unload PrgEnv-pgi PrgEnv-intel PrgEnv-gnu darshan; module load PrgEnv-intel; module unload netcdf intel; module load intel/18.0.6.288 cray-hdf5 cray-netcdf
+    - make work/local-env
+    - make -s -j
+    - (echo '#!/bin/bash';echo '. ./work/local-env/bin/activate';echo 'make MPIRUN="srun -mblock --exclusive" test -s -j') > job.sh
+    - sbatch --clusters=c3,c4 --nodes=5 --time=0:05:00 --account=gfdl_o --qos=debug --job-name=MOM6.gnu.testing --output=log.$CI_PIPELINE_ID --wait job.sh || cat log.$CI_PIPELINE_ID && make test
+
 # Tests
 gnu:non-symmetric:
   stage: tests

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -1,11 +1,16 @@
-# don't build extra formats (like HTML zip)
-formats:
-  - none
+version: 2
 
-# path to pip requirements file to bring in
-# doxygen extensions
-requirements_file: docs/requirements.txt
+# Extra formats
+# PDF generation is failing for now; disabled on 2020-12-02
+#formats:
+#  - pdf
+
+# Build documentation
+sphinx:
+  configuration: docs/conf.py
 
 python:
   # make sure we're using Python 3
-  version: 3.5
+  version: 3
+  install:
+    - requirements: docs/requirements.txt