ufs-community · junwang-noaa · Nov 19, 2020 · Oct 30, 2020 · Oct 31, 2020 · Nov 1, 2020
diff --git a/modulefiles/hera.gnu/fv3 b/modulefiles/hera.gnu/fv3
@@ -40,3 +40,5 @@ module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
+
+module load nccmp/1.8.7.0
diff --git a/modulefiles/hera.intel/fv3 b/modulefiles/hera.intel/fv3
@@ -40,3 +40,5 @@ module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
+
+module load nccmp/1.8.7.0
diff --git a/modulefiles/hera.intel/fv3_debug b/modulefiles/hera.intel/fv3_debug
@@ -40,3 +40,5 @@ module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
+
+module load nccmp/1.8.7.0
diff --git a/modulefiles/orion.intel/fv3 b/modulefiles/orion.intel/fv3
@@ -39,3 +39,5 @@ module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
+
+module load nccmp/1.8.7.0
diff --git a/modulefiles/orion.intel/fv3_debug b/modulefiles/orion.intel/fv3_debug
@@ -39,3 +39,5 @@ module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
+
+module load nccmp/1.8.7.0
diff --git a/modulefiles/wcoss_dell_p3/fv3 b/modulefiles/wcoss_dell_p3/fv3
@@ -43,3 +43,5 @@ module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
+
+module load nccmp/1.8.7.0
diff --git a/modulefiles/wcoss_dell_p3/fv3_debug b/modulefiles/wcoss_dell_p3/fv3_debug
@@ -43,3 +43,5 @@ module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
+
+module load nccmp/1.8.7.0
diff --git a/tests/rt.sh b/tests/rt.sh
@@ -56,6 +56,19 @@ rt_single() {
   fi
 }
 
+rt_35d() {
+  local sy=$(echo ${DATE_35D} | cut -c 1-4)
+  local sm=$(echo ${DATE_35D} | cut -c 5-6)
+  local new_test_name="tests/${TEST_NAME}_${DATE_35D}"
+  rm -f $new_test_name
+  cp tests/$TEST_NAME $new_test_name
+
+  sed -i -e "s/\(export SYEAR\)/\1=\"$sy\"/" $new_test_name
+  sed -i -e "s/\(export SMONTH\)/\1=\"$sm\"/" $new_test_name
+
+  TEST_NAME=${new_test_name#tests/}
+}
+
 rt_trap() {
   [[ ${ROCOTO:-false} == true ]] && rocoto_kill
   [[ ${ECFLOW:-false} == true ]] && ecflow_kill
@@ -117,6 +130,7 @@ if [[ $MACHINE_ID = wcoss_cray ]]; then
   module load ecflow/intel/4.7.1
   ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh
   ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}')
+  NCCMP=''
 
   DISKNM=/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT
   QUEUE=debug
@@ -155,6 +169,7 @@ elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then
   module load ecflow/4.7.1
   ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh
   ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}')
+  NCCMP=$(which nccmp)
 
   DISKNM=/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT
   QUEUE=debug
@@ -174,6 +189,7 @@ elif [[ $MACHINE_ID = gaea.* ]]; then
 #  export PATH=/gpfs/hps/nco/ops/ecf/ecfdir/ecflow.v4.1.0.intel/bin:$PATH
   export PYTHONPATH=
   ECFLOW_START=
+  NCCMP=''
   # DH* 20190717 temporary
   #DISKNM=/lustre/f2/pdata/ncep_shared/emc.nemspara/RT
   DISKNM=/lustre/f2/pdata/esrl/gsd/ufs/ufs-weather-model/RT
@@ -207,6 +223,7 @@ elif [[ $MACHINE_ID = hera.* ]]; then
   export PYTHONPATH=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ecflow/lib/python2.7/site-packages
   ECFLOW_START=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ecflow/bin/ecflow_start.sh
   ECF_PORT=$(( $(id -u) + 1500 ))
+  NCCMP=$(which nccmp)
 
   QUEUE=batch
   COMPILE_QUEUE=batch
@@ -238,6 +255,8 @@ elif [[ $MACHINE_ID = orion.* ]]; then
   export PYTHONPATH=/work/noaa/fv3-cam/djovic/ecflow/lib/python2.7/site-packages
   ECFLOW_START=/work/noaa/fv3-cam/djovic/ecflow/bin/ecflow_start.sh
   ECF_PORT=$(( $(id -u) + 1500 ))
+  NCCMP=$(which nccmp)
+
   QUEUE=batch
   COMPILE_QUEUE=batch
 #  ACCNR= # detected in detect_machine.sh
@@ -268,6 +287,8 @@ elif [[ $MACHINE_ID = jet.* ]]; then
   export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/ecFlow-5.3.1/lib/python2.7/site-packages
   ECFLOW_START=/lfs4/HFIP/hfv3gfs/software/ecFlow-5.3.1/bin/ecflow_start.sh
   ECF_PORT=$(( $(id -u) + 1500 ))
+  NCCMP=''
+
   QUEUE=batch
   COMPILE_QUEUE=batch
   ACCNR=hfv3gfs
@@ -290,6 +311,8 @@ elif [[ $MACHINE_ID = cheyenne.* ]]; then
   export PYTHONPATH=/glade/p/ral/jntp/tools/ecFlow-5.3.1/lib/python2.7/site-packages
   ECFLOW_START=/glade/p/ral/jntp/tools/ecFlow-5.3.1/bin/ecflow_start.sh
   ECF_PORT=$(( $(id -u) + 1500 ))
+  NCCMP=''
+
   QUEUE=regular
   COMPILE_QUEUE=regular
   PARTITION=
@@ -307,6 +330,7 @@ elif [[ $MACHINE_ID = stampede.* ]]; then
 
   export PYTHONPATH=
   ECFLOW_START=
+  NCCMP=''
   QUEUE=skx-dev
   COMPILE_QUEUE=skx-dev
   PARTITION=
@@ -340,6 +364,7 @@ ROCOTO=false
 ECFLOW=false
 KEEP_RUNDIR=false
 SINGLE_NAME=''
+TEST_35D=false
 
 TESTS_FILE='rt.conf'
 
@@ -399,6 +424,10 @@ if [[ $SINGLE_NAME != '' ]]; then
   rt_single
 fi
 
+if [[ $TESTS_FILE =~ '35d' ]]; then
+  TEST_35D=true
+fi
+
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = jet.* ]]; then
   RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20201022/${RT_COMPILER^^}}
 else
@@ -635,11 +664,16 @@ EOF
     MACHINES=$( echo $line | cut -d'|' -f4)
     CB=$(       echo $line | cut -d'|' -f5)
     DEP_RUN=$(  echo $line | cut -d'|' -f6 | sed -e 's/^ *//' -e 's/ *$//')
+    DATE_35D=$( echo $line | cut -d'|' -f7 | sed -e 's/^ *//' -e 's/ *$//')
+
     [[ -e "tests/$TEST_NAME" ]] || die "run test file tests/$TEST_NAME does not exist"
     [[ $SET_ID != ' ' && $SET != *${SET_ID}* ]] && continue
     [[ $MACHINES != ' ' && $MACHINES != *${MACHINE_ID}* ]] && continue
     [[ $CREATE_BASELINE == true && $CB != *fv3* ]] && continue
 
+    # 35 day tests
+    [[ $TEST_35D == true ]] && rt_35d
+
     # skip all *_appbuild runs if rocoto or ecFlow is used. FIXME
     if [[ ${ROCOTO} == true && ${ECFLOW} == true ]]; then
       if [[ ${TEST_NAME} == *_appbuild ]]; then
@@ -685,6 +719,7 @@ EOF
       export PARTITION=${PARTITION}
       export ROCOTO=${ROCOTO}
       export ECFLOW=${ECFLOW}
+      export NCCMP=${NCCMP}
       export REGRESSIONTEST_LOG=${REGRESSIONTEST_LOG}
       export LOG_DIR=${LOG_DIR}
       export DEP_RUN=${DEP_RUN}
@@ -747,6 +782,7 @@ else
   rm -f fv3_*.x fv3_*.exe modules.fv3_*
   [[ ${KEEP_RUNDIR} == false ]] && rm -rf ${RUNDIR_ROOT}
   [[ ${ROCOTO} == true ]] && rm -f ${ROCOTO_XML} ${ROCOTO_DB} *_lock.db
+  [[ ${TEST_35D} == true ]] && rm -f tests/cpld_bmark*_20*
 fi
 
 date >> ${REGRESSIONTEST_LOG}

diff --git a/tests/rt_35d.conf b/tests/rt_35d.conf
@@ -0,0 +1,9 @@
+COMPILE | CCPP=Y SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled S2S=Y  |   | hera.intel orion.intel | fv3 | |
+RUN     | cpld_bmark_35d |   |             | fv3 |  | 2012010100
+RUN     | cpld_bmark_35d |   |             | fv3 |  | 2012040100
+RUN     | cpld_bmark_35d |   |             | fv3 |  | 2012070100
+RUN     | cpld_bmark_35d |   |             | fv3 |  | 2012100100
+RUN     | cpld_bmark_35d |   |             | fv3 |  | 2013010100
+RUN     | cpld_bmark_35d |   |             | fv3 |  | 2013040100
+RUN     | cpld_bmark_35d |   |             | fv3 |  | 2013070100
+RUN     | cpld_bmark_35d |   |             | fv3 |  | 2013100100
diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh
@@ -270,9 +270,25 @@ check_results() {
         d=$( cmp ${RTPWD}/${CNTL_DIR}/$i ${RUNDIR}/$i | wc -l )
 
         if [[ $d -ne 0 ]] ; then
-          echo ".......NOT OK" >> ${REGRESSIONTEST_LOG}
-          echo ".......NOT OK"
-          test_status='FAIL'
+
+          if [[ -n ${NCCMP} ]]; then
+            printf ".......NOT OK...TRY NCCMP.." >> ${REGRESSIONTEST_LOG}
+            printf ".......NOT OK...TRY NCCMP.."
+            d=$( ${NCCMP} -d ${RTPWD}/${CNTL_DIR}/$i ${RUNDIR}/$i 2>&1 | wc -l )
+            if [[ $d -ne 0 ]]; then
+              echo ".....NOT OK" >> ${REGRESSIONTEST_LOG}
+              echo ".....NOT OK"
+              test_status='FAIL'
+            else
+              echo "....OK" >> ${REGRESSIONTEST_LOG}
+              echo "....OK"
+            fi
+          else
+            echo ".......NOT OK" >> ${REGRESSIONTEST_LOG}
+            echo ".......NOT OK"
+            test_status='FAIL'
+          fi
+
         else
           echo "....OK" >> ${REGRESSIONTEST_LOG}
           echo "....OK"

diff --git a/tests/rt_wave_35d.conf b/tests/rt_wave_35d.conf
@@ -0,0 +1,9 @@
+COMPILE | CCPP=Y SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled S2S=Y WW3=Y  |   | hera.intel orion.intel | fv3 | |
+RUN     | cpld_bmark_wave_35d |   |             | fv3 |  | 2012010100
+RUN     | cpld_bmark_wave_35d |   |             | fv3 |  | 2012040100
+RUN     | cpld_bmark_wave_35d |   |             | fv3 |  | 2012070100
+RUN     | cpld_bmark_wave_35d |   |             | fv3 |  | 2012100100
+RUN     | cpld_bmark_wave_35d |   |             | fv3 |  | 2013010100
+RUN     | cpld_bmark_wave_35d |   |             | fv3 |  | 2013040100
+RUN     | cpld_bmark_wave_35d |   |             | fv3 |  | 2013070100
+RUN     | cpld_bmark_wave_35d |   |             | fv3 |  | 2013100100
diff --git a/tests/tests/cpld_2threads b/tests/tests/cpld_2threads
@@ -51,8 +51,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/sfc_data.tile5.nc \
                    RESTART/sfc_data.tile6.nc \
                    RESTART/MOM.res.nc \
-                   RESTART/iced.2016-10-04-00000.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
+                   RESTART/iced.2016-10-04-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_bmark b/tests/tests/cpld_bmark
@@ -53,8 +53,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/MOM.res_1.nc \
                    RESTART/MOM.res_2.nc \
                    RESTART/MOM.res_3.nc \
-                   RESTART/iced.2013-04-02-00000.nc" 
-                   #RESTART/ufs.cpld.cpl.r.2013-04-02-00000.nc"
+                   RESTART/iced.2013-04-02-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2013-04-02-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_bmark_wave b/tests/tests/cpld_bmark_wave
@@ -56,8 +56,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/MOM.res_1.nc \
                    RESTART/MOM.res_2.nc \
                    RESTART/MOM.res_3.nc \
-                   RESTART/iced.2013-04-02-00000.nc"
-                   #RESTART/ufs.cpld.cpl.r.2013-04-02-00000.nc"
+                   RESTART/iced.2013-04-02-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2013-04-02-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_ca b/tests/tests/cpld_ca
@@ -51,8 +51,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/sfc_data.tile5.nc \
                    RESTART/sfc_data.tile6.nc \
                    RESTART/MOM.res.nc \
-                   RESTART/iced.2016-10-04-00000.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
+                   RESTART/iced.2016-10-04-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_control b/tests/tests/cpld_control
@@ -51,8 +51,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/sfc_data.tile5.nc \
                    RESTART/sfc_data.tile6.nc \
                    RESTART/MOM.res.nc \
-                   RESTART/iced.2016-10-04-00000.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
+                   RESTART/iced.2016-10-04-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_control_12h b/tests/tests/cpld_control_12h
@@ -51,8 +51,8 @@ export LIST_FILES="phyf012.tile1.nc \
                    RESTART/sfc_data.tile5.nc \
                    RESTART/sfc_data.tile6.nc \
                    RESTART/MOM.res.nc \
-                   RESTART/iced.2016-10-03-43200.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-03-43200.nc"
+                   RESTART/iced.2016-10-03-43200.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-03-43200.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_control_c192 b/tests/tests/cpld_control_c192
@@ -50,8 +50,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/sfc_data.tile5.nc \
                    RESTART/sfc_data.tile6.nc \
                    RESTART/MOM.res.nc \
-                   RESTART/iced.2016-10-04-00000.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
+                   RESTART/iced.2016-10-04-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_control_c384 b/tests/tests/cpld_control_c384
@@ -53,8 +53,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/MOM.res_1.nc \
                    RESTART/MOM.res_2.nc \
                    RESTART/MOM.res_3.nc \
-                   RESTART/iced.2016-10-04-00000.nc"
-                  #RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
+                   RESTART/iced.2016-10-04-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_control_mx025 b/tests/tests/cpld_control_mx025
@@ -53,8 +53,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/MOM.res_1.nc \
                    RESTART/MOM.res_2.nc \
                    RESTART/MOM.res_3.nc \
-                   RESTART/iced.2016-10-04-00000.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
+                   RESTART/iced.2016-10-04-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_control_mx025_12h b/tests/tests/cpld_control_mx025_12h
@@ -53,8 +53,8 @@ export LIST_FILES="phyf012.tile1.nc \
                    RESTART/MOM.res_1.nc \
                    RESTART/MOM.res_2.nc \
                    RESTART/MOM.res_3.nc \
-                   RESTART/iced.2016-10-03-43200.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-03-43200.nc"
+                   RESTART/iced.2016-10-03-43200.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-03-43200.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_controlfrac_c384 b/tests/tests/cpld_controlfrac_c384
@@ -53,8 +53,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/MOM.res_1.nc \
                    RESTART/MOM.res_2.nc \
                    RESTART/MOM.res_3.nc \
-                   RESTART/iced.2016-10-04-00000.nc"
-                  #RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
+                   RESTART/iced.2016-10-04-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_debug b/tests/tests/cpld_debug
@@ -51,8 +51,8 @@ export LIST_FILES="phyf006.tile1.nc \
                    RESTART/sfc_data.tile5.nc \
                    RESTART/sfc_data.tile6.nc \
                    RESTART/MOM.res.nc \
-                   RESTART/iced.2016-10-03-21600.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-03-21600.nc"
+                   RESTART/iced.2016-10-03-21600.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-03-21600.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_decomp b/tests/tests/cpld_decomp
@@ -51,8 +51,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/sfc_data.tile5.nc \
                    RESTART/sfc_data.tile6.nc \
                    RESTART/MOM.res.nc \
-                   RESTART/iced.2016-10-04-00000.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
+                   RESTART/iced.2016-10-04-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_restart b/tests/tests/cpld_restart
@@ -51,8 +51,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/sfc_data.tile5.nc \
                    RESTART/sfc_data.tile6.nc \
                    RESTART/MOM.res.nc \
-                   RESTART/iced.2016-10-04-00000.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
+                   RESTART/iced.2016-10-04-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
 
 export_fv3
 export_cpl

diff --git a/tests/tests/cpld_restart_mx025 b/tests/tests/cpld_restart_mx025
@@ -54,8 +54,8 @@ export LIST_FILES="phyf024.tile1.nc \
                    RESTART/MOM.res_1.nc \
                    RESTART/MOM.res_2.nc \
                    RESTART/MOM.res_3.nc \
-                   RESTART/iced.2016-10-04-00000.nc"
-                   #RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
+                   RESTART/iced.2016-10-04-00000.nc \
+                   RESTART/ufs.cpld.cpl.r.2016-10-04-00000.nc"
 
 export_fv3
 export_cpl