release/public-v2: b4b reproducibility for restart runs (#417)

* Add coldstart and warmstart test for global fv3_ccpp_gfs_v15p2 * coldstart and warmstart test for global fv3_ccpp_rrfs_v1alpha * Update .gitmodules and submodule pointer for fv3atm for code review and testing * Add regional restart tests and update regression test baseline date tag * Regression test log for orion.intel, hera.intel, cheyenne.intel, cheyenne.gnu,jet.intel, gaea.intel
ufs-community · Feb 16, 2021 · 71b9974 · 71b9974
1 parent 58cf07a
commit 71b9974
Show file tree

Hide file tree

Showing 21 changed files with 1,819 additions and 418 deletions.
diff --git a/FV3 b/FV3
diff --git a/tests/RegressionTests_cheyenne.gnu.log b/tests/RegressionTests_cheyenne.gnu.log
diff --git a/tests/RegressionTests_cheyenne.intel.log b/tests/RegressionTests_cheyenne.intel.log
diff --git a/tests/RegressionTests_gaea.intel.log b/tests/RegressionTests_gaea.intel.log
diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log
diff --git a/tests/RegressionTests_jet.intel.log b/tests/RegressionTests_jet.intel.log
diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log
diff --git a/tests/fv3_conf/ccpp_gfs_v15_run.IN b/tests/fv3_conf/ccpp_gfs_v15_run.IN
@@ -1,11 +1,14 @@
 rm -fr INPUT RESTART
+mkdir INPUT RESTART
 if [ $WARM_START = .F. ]; then
-  cp -r @[RTPWD]/FV3_input_data/INPUT .
-  mkdir RESTART
+  cp -r @[RTPWD]/FV3_input_data/INPUT/* INPUT/
 else
-  mkdir INPUT RESTART
-  cp -r @[RTPWD]/FV3_input_data/RESTART/* ./INPUT
+  cp -r @[RTPWD]/FV3_input_data/INPUT/grid_spec*.nc INPUT/
+  cp -r @[RTPWD]/FV3_input_data/INPUT/C96_grid*.nc INPUT/
+  cp -r @[RTPWD]/FV3_input_data/INPUT/oro_data*.nc INPUT/
+  cp ../fv3_ccpp_gfs_v15p2_coldstart${RT_SUFFIX}/RESTART/* INPUT/
 fi
+
 cp    @[RTPWD]/FV3_input_data/INPUT/aerosol.dat .
 cp    @[RTPWD]/FV3_input_data/INPUT/co2historicaldata_201*.txt .
 cp    @[RTPWD]/FV3_input_data/INPUT/sfc_emissivity_idx.txt .

diff --git a/tests/fv3_conf/ccpp_regional_run.IN b/tests/fv3_conf/ccpp_regional_run.IN
@@ -2,9 +2,18 @@ rm -fr INPUT RESTART
 mkdir INPUT RESTART
 rsync -av @[RTPWD]/FV3_input_data_regional/ ./
 if [ $WARM_START = .T. ]; then
-  cp ../fv3_ccpp_regional_coldstart${RT_SUFFIX}/RESTART/* INPUT/
-  cp ../fv3_ccpp_regional_coldstart${RT_SUFFIX}/INPUT/gfs_bndy.tile7.*.nc INPUT/
-  cp ../fv3_ccpp_regional_coldstart${RT_SUFFIX}/INPUT/oro_data.tile7.halo4.nc INPUT/
+  if [ $CCPP_SUITE = FV3_RRFS_v1alpha ]; then
+    cp ../fv3_ccpp_regional_coldstart${RT_SUFFIX}/RESTART/* INPUT/
+    cp ../fv3_ccpp_regional_coldstart${RT_SUFFIX}/INPUT/gfs_bndy.tile7.*.nc INPUT/
+    cp ../fv3_ccpp_regional_coldstart${RT_SUFFIX}/INPUT/oro_data.tile7.halo4.nc INPUT/
+  elif [ $CCPP_SUITE = FV3_GFS_v15p2 ]; then
+    cp ../fv3_ccpp_regional_v15p2_coldstart${RT_SUFFIX}/RESTART/* INPUT/
+    cp ../fv3_ccpp_regional_v15p2_coldstart${RT_SUFFIX}/INPUT/gfs_bndy.tile7.*.nc INPUT/
+    cp ../fv3_ccpp_regional_v15p2_coldstart${RT_SUFFIX}/INPUT/oro_data.tile7.halo4.nc INPUT/
+  else
+    echo "ERROR, warm start runs not configured for CCPP_SUITE=${CCPP_SUITE}"
+    exit 1
+  fi
 fi
 
 # Copy field table, depending on microphysics choice

diff --git a/tests/rt.conf b/tests/rt.conf
@@ -9,21 +9,23 @@ RUN     | fv3_ccpp_regional_control
 RUN     | fv3_ccpp_regional_decomp                                               | standard    |                |             |
 RUN     | fv3_ccpp_regional_2threads                                             | standard    |                |             |
 RUN     | fv3_ccpp_regional_coldstart                                            | standard    |                |             |
-# The following test does NOT reproduce the control baseline
-#RUN     | fv3_ccpp_regional_warmstart                                            | standard    |                |             | fv3_ccpp_regional_coldstart
+RUN     | fv3_ccpp_regional_warmstart                                            | standard    |                |             | fv3_ccpp_regional_coldstart
 RUN     | fv3_ccpp_regional_v15p2                                                | standard    |                | fv3         |
 RUN     | fv3_ccpp_regional_v15p2_2threads                                       | standard    |                |             |
 # This test only reproduces in REPRO mode (because of GFDL-MP fast physics in 32-bit)
 #RUN     | fv3_ccpp_regional_v15p2_decomp                                         | standard    |                |             |
+RUN     | fv3_ccpp_regional_v15p2_coldstart                                      | standard    |                |             |
+RUN     | fv3_ccpp_regional_v15p2_warmstart                                      | standard    |                |             | fv3_ccpp_regional_coldstart
 
 # Global tests
 RUN     | fv3_ccpp_rrfs_v1alpha                                                  | standard    |                | fv3         |
 RUN     | fv3_ccpp_rrfs_v1alpha_decomp                                           | standard    |                |             |
 RUN     | fv3_ccpp_rrfs_v1alpha_2threads                                         | standard    |                |             |
 RUN     | fv3_ccpp_rrfs_v1alpha_coldstart                                        | standard    |                |             |
-# The following test does NOT reproduce the control baseline
-#RUN     | fv3_ccpp_rrfs_v1alpha_warmstart                                        | standard    |                |             | fv3_ccpp_rrfs_v1alpha_coldstart
+RUN     | fv3_ccpp_rrfs_v1alpha_warmstart                                        | standard    |                |             | fv3_ccpp_rrfs_v1alpha_coldstart
 RUN     | fv3_ccpp_gfs_v15p2                                                     | standard    |                | fv3         |
+RUN     | fv3_ccpp_gfs_v15p2_coldstart                                           | standard    |                |             |
+RUN     | fv3_ccpp_gfs_v15p2_warmstart                                           | standard    |                |             | fv3_ccpp_gfs_v15p2_coldstart
 
 ###############################################################################################################################
 # CCPP REPRO TESTS                                                                                                            #

diff --git a/tests/rt.sh b/tests/rt.sh
@@ -413,9 +413,9 @@ if [[ $SINGLE_NAME != '' ]]; then
 fi
 
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]]; then
-  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/ufs-public-release-v2-20210208/${COMPILER^^}}
+  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/ufs-public-release-v2-20210212/${COMPILER^^}}
 else
-  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/ufs-public-release-v2-20210208}
+  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/ufs-public-release-v2-20210212}
 fi
 
 shift $((OPTIND-1))

diff --git a/tests/tests/fv3_ccpp_gfs_v15p2 b/tests/tests/fv3_ccpp_gfs_v15p2
@@ -20,6 +20,12 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \
                    phyf000.tile4.nc \
                    phyf000.tile5.nc \
                    phyf000.tile6.nc \
+                   phyf012.tile1.nc \
+                   phyf012.tile2.nc \
+                   phyf012.tile3.nc \
+                   phyf012.tile4.nc \
+                   phyf012.tile5.nc \
+                   phyf012.tile6.nc \
                    phyf024.tile1.nc \
                    phyf024.tile2.nc \
                    phyf024.tile3.nc \
@@ -32,6 +38,12 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \
                    dynf000.tile4.nc \
                    dynf000.tile5.nc \
                    dynf000.tile6.nc \
+                   dynf012.tile1.nc \
+                   dynf012.tile2.nc \
+                   dynf012.tile3.nc \
+                   dynf012.tile4.nc \
+                   dynf012.tile5.nc \
+                   dynf012.tile6.nc \
                    dynf024.tile1.nc \
                    dynf024.tile2.nc \
                    dynf024.tile3.nc \
@@ -75,6 +87,8 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \
 export_fv3
 export NODES=$(expr $TASKS / $TPN + 1)
 
+export FHMAX=24
+
 DT_ATMOS="1200"
 
 # Ozone / stratospheric H2O

diff --git a/tests/tests/fv3_ccpp_gfs_v15p2_coldstart b/tests/tests/fv3_ccpp_gfs_v15p2_coldstart
@@ -0,0 +1,54 @@
+###############################################################################
+#
+#  FV3 CCPP GFS v15.2 compiled with 32-bit dynamics test
+#
+###############################################################################
+
+export TEST_DESCR="Compare FV3 32bit CCPP GFS v15.2 results with previous trunk version"
+
+export CNTL_DIR=fv3_gfs_v15p2
+
+export LIST_FILES="phyf000.tile1.nc \
+                   phyf000.tile2.nc \
+                   phyf000.tile3.nc \
+                   phyf000.tile4.nc \
+                   phyf000.tile5.nc \
+                   phyf000.tile6.nc \
+                   phyf012.tile1.nc \
+                   phyf012.tile2.nc \
+                   phyf012.tile3.nc \
+                   phyf012.tile4.nc \
+                   phyf012.tile5.nc \
+                   phyf012.tile6.nc \
+                   dynf000.tile1.nc \
+                   dynf000.tile2.nc \
+                   dynf000.tile3.nc \
+                   dynf000.tile4.nc \
+                   dynf000.tile5.nc \
+                   dynf000.tile6.nc \
+                   dynf012.tile1.nc \
+                   dynf012.tile2.nc \
+                   dynf012.tile3.nc \
+                   dynf012.tile4.nc \
+                   dynf012.tile5.nc \
+                   dynf012.tile6.nc"
+
+
+export_fv3
+export NODES=$(expr $TASKS / $TPN + 1)
+
+export FHMAX=12
+
+DT_ATMOS="1200"
+
+# Ozone / stratospheric H2O
+export OZ_PHYS_OLD=.F.
+export OZ_PHYS_NEW=.T.
+export H2O_PHYS=.T.
+
+export FV3_RUN=ccpp_gfs_v15_run.IN
+export CCPP_SUITE=FV3_GFS_v15p2
+export CCPP_LIB_DIR=ccpp/lib
+export INPUT_NML=ccpp_global.nml.IN
+
+RUN_SCRIPT=rt_fv3.sh
diff --git a/tests/tests/fv3_ccpp_gfs_v15p2_warmstart b/tests/tests/fv3_ccpp_gfs_v15p2_warmstart
@@ -0,0 +1,82 @@
+###############################################################################
+#
+#  FV3 CCPP GFS v15.2 compiled with 32-bit dynamics test
+#
+###############################################################################
+
+export TEST_DESCR="Compare FV3 32bit CCPP GFS v15.2 results with previous trunk version"
+
+export CNTL_DIR=fv3_gfs_v15p2
+
+export LIST_FILES="phyf024.tile1.nc \
+                   phyf024.tile2.nc \
+                   phyf024.tile3.nc \
+                   phyf024.tile4.nc \
+                   phyf024.tile5.nc \
+                   phyf024.tile6.nc \
+                   dynf024.tile1.nc \
+                   dynf024.tile2.nc \
+                   dynf024.tile3.nc \
+                   dynf024.tile4.nc \
+                   dynf024.tile5.nc \
+                   dynf024.tile6.nc \
+                   RESTART/coupler.res \
+                   RESTART/fv_core.res.nc \
+                   RESTART/fv_core.res.tile1.nc \
+                   RESTART/fv_core.res.tile2.nc \
+                   RESTART/fv_core.res.tile3.nc \
+                   RESTART/fv_core.res.tile4.nc \
+                   RESTART/fv_core.res.tile5.nc \
+                   RESTART/fv_core.res.tile6.nc \
+                   RESTART/fv_srf_wnd.res.tile1.nc \
+                   RESTART/fv_srf_wnd.res.tile2.nc \
+                   RESTART/fv_srf_wnd.res.tile3.nc \
+                   RESTART/fv_srf_wnd.res.tile4.nc \
+                   RESTART/fv_srf_wnd.res.tile5.nc \
+                   RESTART/fv_srf_wnd.res.tile6.nc \
+                   RESTART/fv_tracer.res.tile1.nc \
+                   RESTART/fv_tracer.res.tile2.nc \
+                   RESTART/fv_tracer.res.tile3.nc \
+                   RESTART/fv_tracer.res.tile4.nc \
+                   RESTART/fv_tracer.res.tile5.nc \
+                   RESTART/fv_tracer.res.tile6.nc \
+                   RESTART/sfc_data.tile1.nc \
+                   RESTART/sfc_data.tile2.nc \
+                   RESTART/sfc_data.tile3.nc \
+                   RESTART/sfc_data.tile4.nc \
+                   RESTART/sfc_data.tile5.nc \
+                   RESTART/sfc_data.tile6.nc \
+                   RESTART/phy_data.tile1.nc \
+                   RESTART/phy_data.tile2.nc \
+                   RESTART/phy_data.tile3.nc \
+                   RESTART/phy_data.tile4.nc \
+                   RESTART/phy_data.tile5.nc \
+                   RESTART/phy_data.tile6.nc"
+
+
+export_fv3
+export NODES=$(expr $TASKS / $TPN + 1)
+
+export WARM_START=.T.
+export NGGPS_IC=.F.
+export EXTERNAL_IC=.F.
+export MAKE_NH=.F.
+export MOUNTAIN=.T.
+export NA_INIT=0
+export NSTF_NAME=2,0,1,0,5
+
+export FHMAX=24
+
+DT_ATMOS="1200"
+
+# Ozone / stratospheric H2O
+export OZ_PHYS_OLD=.F.
+export OZ_PHYS_NEW=.T.
+export H2O_PHYS=.T.
+
+export FV3_RUN=ccpp_gfs_v15_run.IN
+export CCPP_SUITE=FV3_GFS_v15p2
+export CCPP_LIB_DIR=ccpp/lib
+export INPUT_NML=ccpp_global.nml.IN
+
+RUN_SCRIPT=rt_fv3.sh
diff --git a/tests/tests/fv3_ccpp_regional_coldstart b/tests/tests/fv3_ccpp_regional_coldstart
@@ -63,4 +63,4 @@ export WRITE_RESTART_WITH_BCS=.true.
 
 if [[ $MACHINE_ID = *.gnu ]]; then
   export WLCLK=60
-fi
+fi
diff --git a/tests/tests/fv3_ccpp_regional_v15p2 b/tests/tests/fv3_ccpp_regional_v15p2
@@ -10,8 +10,10 @@ export CNTL_DIR=fv3_regional_v15p2
 
 export LIST_FILES="atmos_4xdaily.nc \
                    phyf000.nc \
+                   phyf006.nc \
                    phyf012.nc \
                    dynf000.nc \
+                   dynf006.nc \
                    dynf012.nc \
                    RESTART/coupler.res \
                    RESTART/fv_core.res.nc \

diff --git a/tests/tests/fv3_ccpp_regional_v15p2_coldstart b/tests/tests/fv3_ccpp_regional_v15p2_coldstart
@@ -0,0 +1,53 @@
+###############################################################################
+#
+#  FV3 CCPP regional control test using GFS v15p2 suite with 32-bit dynamics
+#
+###############################################################################
+
+export TEST_DESCR="Compare FV3 CCPP regional GFS v15p2 results with previous trunk version"
+
+export CNTL_DIR=fv3_regional_v15p2
+
+export LIST_FILES="phyf000.nc \
+                   phyf006.nc \
+                   dynf000.nc \
+                   dynf006.nc"
+
+export_fv3
+export TASKS=80
+export NODES=$(expr $TASKS / $TPN + 1)
+
+export INPES=20
+export JNPES=2
+
+export FHMAX=6
+export FDIAG=3
+
+DT_ATMOS="300"
+
+export FV3_RUN=ccpp_regional_run.IN
+export CCPP_SUITE=FV3_GFS_v15p2
+export CCPP_LIB_DIR=ccpp/lib
+export INPUT_NML=ccpp_regional.nml.IN
+export MODEL_CONFIGURE=ccpp_regional-model_configure.IN
+
+# Ozone / stratospheric H2O
+export OZ_PHYS_OLD=.F.
+export OZ_PHYS_NEW=.T.
+export H2O_PHYS=.T.
+
+# GFDL microphysics
+export IMP_PHYSICS=11
+
+# Other namelist switches
+export HYBEDMF=.T.
+export FHCYC=0
+
+RUN_SCRIPT=rt_fv3.sh
+
+# Regional option
+export WRITE_RESTART_WITH_BCS=.true.
+
+if [[ $MACHINE_ID = *.gnu ]]; then
+  export WLCLK=60
+fi
diff --git a/tests/tests/fv3_ccpp_regional_v15p2_warmstart b/tests/tests/fv3_ccpp_regional_v15p2_warmstart
@@ -0,0 +1,66 @@
+###############################################################################
+#
+#  FV3 CCPP regional control test using GFS v15p2 suite with 32-bit dynamics
+#
+###############################################################################
+
+export TEST_DESCR="Compare FV3 CCPP regional GFS v15p2 results with previous trunk version"
+
+export CNTL_DIR=fv3_regional_v15p2
+
+export LIST_FILES="phyf012.nc \
+                   dynf012.nc \
+                   RESTART/coupler.res \
+                   RESTART/fv_core.res.nc \
+                   RESTART/fv_core.res.tile1.nc \
+                   RESTART/fv_srf_wnd.res.tile1.nc \
+                   RESTART/fv_tracer.res.tile1.nc \
+                   RESTART/sfc_data.nc \
+                   RESTART/phy_data.nc"
+
+export_fv3
+export TASKS=80
+export NODES=$(expr $TASKS / $TPN + 1)
+
+export INPES=20
+export JNPES=2
+
+export WARM_START=.T.
+export NGGPS_IC=.F.
+export EXTERNAL_IC=.F.
+export MAKE_NH=.F.
+export MOUNTAIN=.T.
+export NA_INIT=0
+export NSTF_NAME=2,0,1,0,5
+
+export FHMAX=12
+export FDIAG=3
+
+DT_ATMOS="300"
+
+export FV3_RUN=ccpp_regional_run.IN
+export CCPP_SUITE=FV3_GFS_v15p2
+export CCPP_LIB_DIR=ccpp/lib
+export INPUT_NML=ccpp_regional.nml.IN
+export MODEL_CONFIGURE=ccpp_regional-model_configure.IN
+
+# Ozone / stratospheric H2O
+export OZ_PHYS_OLD=.F.
+export OZ_PHYS_NEW=.T.
+export H2O_PHYS=.T.
+
+# GFDL microphysics
+export IMP_PHYSICS=11
+
+# Other namelist switches
+export HYBEDMF=.T.
+export FHCYC=0
+
+RUN_SCRIPT=rt_fv3.sh
+
+# Regional option
+export WRITE_RESTART_WITH_BCS=.true.
+
+if [[ $MACHINE_ID = *.gnu ]]; then
+  export WLCLK=60
+fi