Clean up the input data directory, move diag_table to ufs-weather-mod…

…el and set up C768 atm-only test (#921)

* The input data directory was cleaned up.
*  The diag_tables used by the RT tests are moved to tests/parm/diag_table. The diag_tables and field_tables under the input data directory are removed. The date in several diag_tables used by global control and related tests are corrected from 2016100300 to the new IC 2021032206. The diag_table for regional_qulit related tests are updated to be consistent with regional_control.
* A high resolution C768 atmosphere only test and a lat/lon control tests were created.
* net lw flux issues was fixed in fv3 and ccpp physics.

Co-authored-by: Dom Heinzeller <climbfuji@ymail.com>

tests/ci/ci.sh

@@ -75,7 +75,7 @@ if [ $BUILD = "true" ]; then
 elif [ $RUN == "true" ]; then
 
   docker volume rm -f DataVolume >/dev/null &&
-    docker run -d --rm -v DataVolume:/tmp minsukjinoaa/input-data:20210930
+    docker run -d --rm -v DataVolume:/tmp minsukjinoaa/input-data:20211203
 
   docker create -u builder -e "CI_TEST=true" -e "USER=builder" \
                 -e "RT_MACHINE=linux.gnu" -e "RT_COMPILER=gnu" \

tests/default_vars.sh

@@ -12,6 +12,7 @@ if [[ $MACHINE_ID = wcoss_cray ]]; then
   TASKS_dflt=150 ; TPN_dflt=24 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=78  ; TPN_thrd=12 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_c384=336 ; TPN_c384=20 ; INPES_c384=8 ; JNPES_c384=6
+  TASKS_c768=928 ; TPN_c768=12 ; INPES_c768=8 ; JNPES_c768=16
   TASKS_stretch=48 ; TPN_stretch=24 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=24 ; INPES_strnest=2 ; JNPES_strnest=4
 
@@ -20,6 +21,7 @@ elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then
   TASKS_dflt=150 ; TPN_dflt=28 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=78  ; TPN_thrd=14 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_c384=336 ; TPN_c384=20 ; INPES_c384=8 ; JNPES_c384=6
+  TASKS_c768=928 ; TPN_c768=14 ; INPES_c768=8 ; JNPES_c768=16
   TASKS_stretch=48 ; TPN_stretch=28 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=28 ; INPES_strnest=2 ; JNPES_strnest=4
 
@@ -87,6 +89,7 @@ elif [[ $MACHINE_ID = wcoss2 ]]; then
   TASKS_dflt=150 ; TPN_dflt=128 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=78  ; TPN_thrd=64 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_c384=336 ; TPN_c384=128 ; INPES_c384=8 ; JNPES_c384=6
+  TASKS_c768=928 ; TPN_c768=32 ; INPES_c768=8 ; JNPES_c768=16
   TASKS_stretch=48 ; TPN_stretch=128 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=128 ; INPES_strnest=2 ; JNPES_strnest=4
 
@@ -154,6 +157,7 @@ elif [[ $MACHINE_ID = orion.* ]]; then
   TASKS_dflt=150 ; TPN_dflt=40 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=78  ; TPN_thrd=20 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_c384=336 ; TPN_c384=20 ; INPES_c384=8 ; JNPES_c384=6
+  TASKS_c768=928 ; TPN_c768=10 ; INPES_c768=8 ; JNPES_c768=16
   TASKS_stretch=48 ; TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=12 ; INPES_strnest=2 ; JNPES_strnest=4
 
@@ -221,6 +225,7 @@ elif [[ $MACHINE_ID = hera.* ]]; then
   TASKS_dflt=150 ; TPN_dflt=40 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=78  ; TPN_thrd=20 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_c384=336 ; TPN_c384=20 ; INPES_c384=6 ; JNPES_c384=8
+  TASKS_c768=928 ; TPN_c768=10 ; INPES_c768=8 ; JNPES_c768=16
   TASKS_stretch=48 ; TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=12 ; INPES_strnest=2 ; JNPES_strnest=4
 
@@ -309,7 +314,8 @@ elif [[ $MACHINE_ID = jet.* ]]; then
 
   TASKS_dflt=150 ; TPN_dflt=24 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=78  ; TPN_thrd=12 ; INPES_thrd=3 ; JNPES_thrd=4
-  TASKS_c384=480 ; TPN_c384=12 ; INPES_c384=12 ; JNPES_c384=6
+  TASKS_c384=480 ; TPN_c384=12 ; INPES_c384=6 ; JNPES_c384=12
+  TASKS_c768=928 ; TPN_c768=6 ;  INPES_c768=8 ; JNPES_c768=16
   TASKS_stretch=48 ; TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=12 ; INPES_strnest=2 ; JNPES_strnest=4
 
@@ -377,6 +383,7 @@ elif [[ $MACHINE_ID = s4.* ]]; then
   TASKS_dflt=150 ; TPN_dflt=32 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=78  ; TPN_thrd=16 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_c384=336 ; TPN_c384=16 ; INPES_c384=6 ; JNPES_c384=8
+  TASKS_c768=928 ; TPN_c768=8  ; INPES_c768=8 ; JNPES_c768=16
   TASKS_stretch=48 ; TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=12 ; INPES_strnest=2 ; JNPES_strnest=4
 
@@ -444,6 +451,7 @@ elif [[ $MACHINE_ID = gaea.* ]]; then
   TASKS_dflt=150 ; TPN_dflt=36 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=78  ; TPN_thrd=18 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_c384=336 ; TPN_c384=18 ; INPES_c384=6 ; JNPES_c384=8
+  TASKS_c768=928 ; TPN_c768=9  ; INPES_c768=8 ; JNPES_c768=16
   TASKS_stretch=48 ; TPN_stretch=18 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=18 ; INPES_strnest=2 ; JNPES_strnest=4
 
@@ -511,6 +519,7 @@ elif [[ $MACHINE_ID = cheyenne.* ]]; then
   TASKS_dflt=150 ; TPN_dflt=36 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=78  ; TPN_thrd=18 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_c384=336 ; TPN_c384=18 ; INPES_c384=8 ; JNPES_c384=6
+  TASKS_c768=992 ; TPN_c768=18 ; INPES_c768=8 ; JNPES_c768=16
   TASKS_stretch=48 ; TPN_stretch=18 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=18 ; INPES_strnest=2 ; JNPES_strnest=4
 
@@ -578,6 +587,7 @@ elif [[ $MACHINE_ID = stampede.* ]]; then
   TASKS_dflt=150 ; TPN_dflt=48 ; INPES_dflt=3 ; JNPES_dflt=8
   TASKS_thrd=78  ; TPN_thrd=24 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_c384=336 ; TPN_c384=20 ; INPES_c384=8 ; JNPES_c384=6
+  TASKS_c768=992 ; TPN_c768=20 ; INPES_c768=8 ; JNPES_c768=16
   TASKS_stretch=48 ; TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4
 
   TASKS_cpl_atmw=192; TPN_cpl_atmw=48; INPES_cpl_atmw=3; JNPES_cpl_atmw=8
@@ -724,8 +734,6 @@ export DATM_CDEPS=false
 export DOCN_CDEPS=false
 export THRD=1
 export POSTAPP='global'
-export NEW_DIAGTABLE='none'
-export NEW_FIELDTABLE='none'
 export USE_MERRA2=.false.
 
 export INPES=$INPES_dflt
@@ -744,10 +752,14 @@ export OUTPUT_GRID="'cubed_sphere_grid'"
 export OUTPUT_FILE="'netcdf'"
 export IDEFLATE=0
 export NBITS=0
+export ICHUNK3D=0
+export JCHUNK3D=0
+export KCHUNK3D=0
 export IMO=384
 export JMO=190
 
 #input file
+export DIAG_TABLE=diag_table_gfsv16
 export FIELD_TABLE=field_table_gfsv16
 
 # Coldstart/warmstart
@@ -1412,6 +1424,8 @@ export DLAT=0.03
 
 # input.nml
 export CPL_IMP_MRG=.true.
+export DIAG_TABLE=diag_table_hafs
+export FIELD_TABLE=field_table_hafs
 
 # nems.configure
 export med_model=cmeps

tests/fv3_conf/c96_HAFS_v0_hwrf_run.IN

@@ -21,29 +21,6 @@ else
   exit 1
 fi
 
-cp    @[INPUTDATA_ROOT]/FV3_input_data/INPUT/aerosol.dat .
-cp    @[INPUTDATA_ROOT]/FV3_input_data/INPUT/co2historicaldata_201*.txt .
-cp    @[INPUTDATA_ROOT]/FV3_input_data/INPUT/sfc_emissivity_idx.txt .
-cp    @[INPUTDATA_ROOT]/FV3_input_data/INPUT/solarconstant_noaa_an.txt .
-cp    @[INPUTDATA_ROOT]/FV3_input_data/ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77 ./global_o3prdlos.f77
-cp    @[INPUTDATA_ROOT]/FV3_input_data/global_h2o_pltc.f77 ./global_h2oprdlos.f77
-cp    @[INPUTDATA_ROOT]/FV3_input_data/*grb .
-cp    @[INPUTDATA_ROOT]/FV3_input_data/diag_table .
-
-# Copy diag table, depending on microphysics choice
-if [ $IMP_PHYSICS = 8 ]; then
-  cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/diag_table_gsd_noah diag_table
-elif [ $IMP_PHYSICS = 15 ]; then
-  if [ $IOVR = 4 ]; then
-    cp @[INPUTDATA_ROOT]/FV3_input_data_hafs/diag_table_FA_HWRF diag_table
-  else
-    cp @[INPUTDATA_ROOT]/FV3_input_data_hafs/diag_table_FA diag_table
-  fi
-else
-  echo "ERROR, no diag table configured for IMP_PHYSICS=${IMP_PHYSICS}"
-  exit 1
-fi
-
 # Thompson or F-A MP lookup tables
 if [ $IMP_PHYSICS = 8 ]; then
   cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qsV2.dat .

tests/fv3_conf/control_run.IN

@@ -8,7 +8,7 @@ elif [ $NPX = 193 ]; then
 elif [ $NPX = 385 ]; then
   inputdir=FV3_input_data384
 elif [ $NPX = 769 ]; then
-  inputdir=FV3_input_data_c768
+  inputdir=FV3_input_data768
 fi
 echo "inputdir=$inputdir,NPX=$NPX"
 
@@ -73,36 +73,18 @@ else
 
 fi
 
-if [ $NPX = 49 ]; then
-    cp    @[INPUTDATA_ROOT]/${inputdir}/aerosol.dat .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/co2historicaldata_201*.txt .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/sfc_emissivity_idx.txt .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/solarconstant_noaa_an.txt .
-else
-    cp    @[INPUTDATA_ROOT]/${inputdir}/INPUT/aerosol.dat .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/INPUT/co2historicaldata_201*.txt .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/INPUT/sfc_emissivity_idx.txt .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/INPUT/solarconstant_noaa_an.txt .
-fi
-cp    @[INPUTDATA_ROOT]/${inputdir}/global_o3prdlos.f77 .
-cp    @[INPUTDATA_ROOT]/${inputdir}/global_h2o_pltc.f77 ./global_h2oprdlos.f77
 if [ $TILEDFIX = .true. ]; then
     cp    @[INPUTDATA_ROOT]/FV3_fix_tiled/@[ATMRES]/@[ATMRES]*.nc .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/global_glacier.2x2.grb .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/global_maxice.2x2.grb .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/RTGSST.1982.2012.monthly.clim.grb .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/global_snoclim.1.875.grb .
-    cp    @[INPUTDATA_ROOT]/${inputdir}/CFSR.SEAICE.1982.2012.monthly.clim.grb .
+    cp    @[INPUTDATA_ROOT]/FV3_fix/global_glacier.2x2.grb .
+    cp    @[INPUTDATA_ROOT]/FV3_fix/global_maxice.2x2.grb .
+    cp    @[INPUTDATA_ROOT]/FV3_fix/RTGSST.1982.2012.monthly.clim.grb .
+    cp    @[INPUTDATA_ROOT]/FV3_fix/global_snoclim.1.875.grb .
+    cp    @[INPUTDATA_ROOT]/FV3_fix/CFSR.SEAICE.1982.2012.monthly.clim.grb .
     cp    @[INPUTDATA_ROOT]/${inputdir}/@[FNSMCC] .
     cp    @[INPUTDATA_ROOT]/${inputdir}/@[FNMSKH_control] .
 else
     cp    @[INPUTDATA_ROOT]/${inputdir}/*grb .
 fi
-cp    @[INPUTDATA_ROOT]/${inputdir}/diag_table_gfsv16 ./diag_table
-
-if [ $NEW_DIAGTABLE != 'none' ]; then
-  cp    @[INPUTDATA_ROOT]/${inputdir}/$NEW_DIAGTABLE ./diag_table
-fi
 
 if [ $CPLWAV = .true. ]; then
   cp    @[INPUTDATA_ROOT_WW3]/mod_def.* .
@@ -149,7 +131,6 @@ fi
 
 #prognostic aerosols
 if [ $CPLCHM = .true. ]; then
-  cp      ${PATHRT}/parm/gocart/diag_table  .
   cp      ${PATHRT}/parm/gocart/*.rc .
   ln -sf  @[INPUTDATA_ROOT]/GOCART/ExtData .
 fi

tests/fv3_conf/cpld_control_run.IN

@@ -19,20 +19,9 @@ else
 fi
 
 # FV3 fixed input
-cp    @[INPUTDATA_ROOT_BMIC]/FV3_fix/INPUT/co2historicaldata_201*.txt .
-cp    @[INPUTDATA_ROOT]/${FV3_DIR}/INPUT/aerosol.dat .
-cp    @[INPUTDATA_ROOT]/${FV3_DIR}/INPUT/sfc_emissivity_idx.txt .
-cp    @[INPUTDATA_ROOT]/${FV3_DIR}/INPUT/solarconstant_noaa_an.txt .
-cp    @[INPUTDATA_ROOT]/${FV3_DIR}/global_glacier.2x2.grb .
-cp    @[INPUTDATA_ROOT]/${FV3_DIR}/global_maxice.2x2.grb .
-cp    @[INPUTDATA_ROOT]/${FV3_DIR}/RTGSST.1982.2012.monthly.clim.grb .
-cp    @[INPUTDATA_ROOT]/${FV3_DIR}/global_snoclim.1.875.grb .
-cp    @[INPUTDATA_ROOT]/${FV3_DIR}/CFSR.SEAICE.1982.2012.monthly.clim.grb .
 cp    @[INPUTDATA_ROOT]/${FV3_DIR}/@[FNSMCC] .
 cp    @[INPUTDATA_ROOT]/${FV3_DIR}/@[FNMSKH] .
-cp    @[INPUTDATA_ROOT]/${FV3_DIR}/global_o3prdlos.f77 .
 
-cp    @[INPUTDATA_ROOT]/${FV3_DIR}/global_h2o_pltc.f77 ./global_h2oprdlos.f77
 cp    @[INPUTDATA_ROOT]/FV3_fix_tiled/@[ATMRES]/oro_@[ATMRES].mx@[OCNRES].tile1.nc INPUT/oro_data.tile1.nc
 cp    @[INPUTDATA_ROOT]/FV3_fix_tiled/@[ATMRES]/oro_@[ATMRES].mx@[OCNRES].tile2.nc INPUT/oro_data.tile2.nc
 cp    @[INPUTDATA_ROOT]/FV3_fix_tiled/@[ATMRES]/oro_@[ATMRES].mx@[OCNRES].tile3.nc INPUT/oro_data.tile3.nc

tests/fv3_conf/esg_HAFS_v0_hwrf_run.IN

@@ -2,22 +2,8 @@ rm -fr INPUT RESTART
 mkdir INPUT RESTART
 
 if [ $WARM_START = .false. ]; then
-  rsync -av --exclude 'field_table' @[INPUTDATA_ROOT]/FV3_input_data_regional_esg/ ./
+  rsync -av @[INPUTDATA_ROOT]/FV3_input_data_regional_esg/ ./
 else
   echo "ERROR, warmstart runs not configured for regional HAFs runs on ESG grid"
   exit 1
 fi
-
-# Copy diag table, depending on microphysics choice
-if [ $IMP_PHYSICS = 8 ]; then
-  ln -sf diag_table.thompson diag_table
-elif [ $IMP_PHYSICS = 15 ]; then
-  if [ $IOVR = 4 ]; then
-    cp @[INPUTDATA_ROOT]/FV3_input_data_hafs/diag_table_FA_HWRF diag_table
-  else
-    cp @[INPUTDATA_ROOT]/FV3_input_data_hafs/diag_table_FA diag_table
-  fi
-else
-  echo "ERROR, no diag table configured for IMP_PHYSICS=${IMP_PHYSICS}"
-  exit 1
-fi

tests/fv3_conf/fv3_gocart.IN

@@ -6,16 +6,7 @@ else
   mkdir INPUT RESTART
   cp -r @[INPUTDATA_ROOT]/FV3_input_data/RESTART/* ./INPUT
 fi
-cp    @[INPUTDATA_ROOT]/FV3_input_data/INPUT/aerosol.dat .
-cp    @[INPUTDATA_ROOT]/FV3_input_data/INPUT/co2historicaldata_201*.txt .
-cp    @[INPUTDATA_ROOT]/FV3_input_data/INPUT/sfc_emissivity_idx.txt .
-cp    @[INPUTDATA_ROOT]/FV3_input_data/INPUT/solarconstant_noaa_an.txt .
-cp    @[INPUTDATA_ROOT]/FV3_input_data/INPUT/global_o3prdlos.f77 .
 cp    @[INPUTDATA_ROOT]/FV3_input_data/*grb .
-cp    @[INPUTDATA_ROOT]/FV3_input_data/*_table .
-##cp    @[INPUTDATA_ROOT]/FV3_input_data/diag_table_gfdlmp diag_table
-cp    @[INPUTDATA_ROOT]/FV3_input_data/diag_table_aod diag_table
-cp    @[INPUTDATA_ROOT]/FV3_input_data/field_table_gfdlmp field_table
 cp    @[INPUTDATA_ROOT]/FV3_input_data_INCCN_aeroclim/MERRA2/merra2C.aerclim.2003-2014.*nc .
 for n in 01 02 03 04 05 06 07 08 09 10 11 12; do
 ln -sf merra2C.aerclim.2003-2014.m${n}.nc  aeroclim.m${n}.nc