Skip to content

Commit

Permalink
Convert all global (uncoupled) and regional tests to use quilting res…
Browse files Browse the repository at this point in the history 
…tart (#1946)

* Convert most global (uncoupled) and regional tests to use quilting restart

* use MOM6 test branch, modify mom6_files.cmake

* update MOM6 to include 20231106 NOAA-GFDL/rtd_fix

* Update cpld_control_p8 remove hera_gnu wall clock stipulation

---------

Co-authored-by: jiandewang <jiande.wang@noaa.gov>
Co-authored-by: WenMeng-NOAA  <wen.meng@noaa.gov>
  • Loading branch information
@DusanJovic-NOAA @jiandewang @WenMeng-NOAA
3 people authored Nov 14, 2023
1 parent 9e41ad9 commit 17643bc
Show file tree
Hide file tree
Showing 37 changed files with 14,754 additions and 18,215 deletions.
2 changes: 1 addition & 1 deletion FV3
Submodule FV3 updated 3 files
+1 −1 fv3_cap.F90
+12 −1 io/post_fv3.F90
+1 −1 upp
2 changes: 1 addition & 1 deletion MOM6-interface/MOM6
Submodule MOM6 updated 37 files
+0 −4 .github/workflows/coupled-api.yml
+9 −2 .readthedocs.yml
+0 −7 .testing/Makefile
+8 −0 config_src/drivers/FMS_cap/ocean_model_MOM.F90
+0 −0 config_src/drivers/STALE_mct_cap/mom_ocean_model_mct.F90
+0 −0 config_src/drivers/STALE_mct_cap/mom_surface_forcing_mct.F90
+0 −0 config_src/drivers/STALE_mct_cap/ocn_cap_methods.F90
+0 −0 config_src/drivers/STALE_mct_cap/ocn_comp_mct.F90
+0 −0 config_src/drivers/STALE_mct_cap/ocn_cpl_indices.F90
+46 −19 config_src/drivers/nuopc_cap/mom_cap.F90
+13 −7 config_src/drivers/nuopc_cap/mom_ocean_model_nuopc.F90
+3 −40 config_src/drivers/nuopc_cap/mom_surface_forcing_nuopc.F90
+2 −1 config_src/drivers/solo_driver/MOM_surface_forcing.F90
+10 −3 config_src/infra/FMS1/MOM_ensemble_manager_infra.F90
+10 −3 config_src/infra/FMS2/MOM_ensemble_manager_infra.F90
+2 −0 docs/requirements.txt
+5 −3 src/core/MOM.F90
+77 −3 src/core/MOM_dynamics_split_RK2.F90
+27 −42 src/core/MOM_forcing_type.F90
+1 −1 src/core/MOM_unit_tests.F90
+2 −13 src/core/MOM_variables.F90
+2 −0 src/diagnostics/MOM_obsolete_params.F90
+8 −1 src/framework/MOM_file_parser.F90
+1 −1 src/framework/MOM_get_input.F90
+345 −34 src/parameterizations/lateral/MOM_hor_visc.F90
+16 −0 src/parameterizations/vertical/MOM_CVMix_KPP.F90
+2 −1 src/parameterizations/vertical/MOM_set_viscosity.F90
+396 −0 src/parameterizations/vertical/MOM_vert_friction.F90
+195 −144 src/tracer/MOM_CFC_cap.F90
+272 −146 src/tracer/MOM_hor_bnd_diffusion.F90
+376 −88 src/tracer/MOM_neutral_diffusion.F90
+7 −4 src/tracer/MOM_tracer_flow_control.F90
+176 −99 src/tracer/MOM_tracer_hor_diff.F90
+31 −43 src/tracer/MOM_tracer_registry.F90
+13 −17 src/tracer/MOM_tracer_types.F90
+1 −1 src/tracer/oil_tracer.F90
+31 −6 src/user/MOM_wave_interface.F90
2 changes: 1 addition & 1 deletion MOM6-interface/mom6_files.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ list(APPEND mom6_src_files

MOM6/src/tracer/MOM_OCMIP2_CFC.F90
MOM6/src/tracer/MOM_generic_tracer.F90
MOM6/src/tracer/MOM_lateral_boundary_diffusion.F90
MOM6/src/tracer/MOM_hor_bnd_diffusion.F90
MOM6/src/tracer/MOM_neutral_diffusion.F90
MOM6/src/tracer/nw2_tracers.F90
MOM6/src/tracer/MOM_offline_aux.F90
Expand Down
7 changes: 5 additions & 2 deletions tests/default_vars.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ elif [[ $MACHINE_ID = jet ]]; then
INPES_thrd=3 ; JNPES_thrd=4
INPES_c384=6 ; JNPES_c384=12 ; THRD_c384=1
INPES_c768=8 ; JNPES_c768=16 ; THRD_c768=2
WRTTASK_PER_GROUP_c384=84
WRTTASK_PER_GROUP_c384gdas=88

THRD_cpl_atmw_gdas=2
INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24
Expand Down Expand Up @@ -357,7 +359,7 @@ export INPES=$INPES_dflt
export JNPES=$JNPES_dflt
export RESTART_INTERVAL=0
export QUILTING=.true.
export QUILTING_RESTART=.false.
export QUILTING_RESTART=.true.
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=6
export ITASKS=1
Expand Down Expand Up @@ -698,6 +700,8 @@ export JNPES=$JNPES_cpl_dflt
export THRD=$THRD_cpl_dflt
export WRTTASK_PER_GROUP=$WPG_cpl_dflt

export QUILTING_RESTART=.false.

OCN_tasks=$OCN_tasks_cpl_dflt
ICE_tasks=$ICE_tasks_cpl_dflt
WAV_tasks=$WAV_tasks_cpl_dflt
Expand Down Expand Up @@ -1147,7 +1151,6 @@ export RESTART_INTERVAL=0
export FHROT=0
export coupling_interval_fast_sec=0
export QUILTING=.true.
export QUILTING_RESTART=.false.
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=6
export OUTPUT_HISTORY=.true.
Expand Down
4,127 changes: 1,879 additions & 2,248 deletions tests/logs/RegressionTests_acorn.log
View file

Large diffs are not rendered by default.

4,081 changes: 1,856 additions & 2,225 deletions tests/logs/RegressionTests_gaea.log
View file

Large diffs are not rendered by default.

5,981 changes: 2,772 additions & 3,209 deletions tests/logs/RegressionTests_hera.log
View file

Large diffs are not rendered by default.

6,055 changes: 2,795 additions & 3,260 deletions tests/logs/RegressionTests_hercules.log
View file

Large diffs are not rendered by default.

3,599 changes: 1,732 additions & 1,867 deletions tests/logs/RegressionTests_jet.log
View file

Large diffs are not rendered by default.

4,386 changes: 2,012 additions & 2,374 deletions tests/logs/RegressionTests_orion.log
View file

Large diffs are not rendered by default.

3,820 changes: 1,678 additions & 2,142 deletions tests/logs/RegressionTests_wcoss2.log
View file

Large diffs are not rendered by default.

20 changes: 4 additions & 16 deletions tests/rt.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ RUN | control_iovr5 |
RUN | control_p8 | - noaacloud | baseline |
RUN | control_p8_ugwpv1 | - noaacloud | baseline |
RUN | control_restart_p8 | - noaacloud | | control_p8
RUN | control_qr_p8 | - noaacloud | |
RUN | control_restart_qr_p8 | - noaacloud | | control_qr_p8
RUN | control_noqr_p8 | - noaacloud | |
RUN | control_restart_noqr_p8 | - noaacloud | | control_noqr_p8
RUN | control_decomp_p8 | - noaacloud | |
RUN | control_2threads_p8 | - noaacloud | |
RUN | control_p8_lndp | | baseline |
Expand All @@ -109,8 +109,6 @@ RUN | control_p8_mynn | - noaacloud
RUN | merra2_thompson | - noaacloud | baseline |
RUN | regional_control | | baseline |
RUN | regional_restart | - noaacloud | | regional_control
RUN | regional_control_qr | - noaacloud | |
RUN | regional_restart_qr | - noaacloud | | regional_control_qr
RUN | regional_decomp | - noaacloud | |
RUN | regional_2threads | - noaacloud | |
RUN | regional_noquilt | - jet s4 | baseline |
Expand All @@ -133,11 +131,9 @@ RUN | rap_sfcdiff | - noaacloud
RUN | rap_sfcdiff_decomp | - noaacloud | |
RUN | rap_sfcdiff_restart | - noaacloud | | rap_sfcdiff
RUN | hrrr_control | - noaacloud | baseline |
RUN | hrrr_control_qr | - noaacloud | |
RUN | hrrr_control_decomp | - noaacloud | |
RUN | hrrr_control_2threads | - noaacloud | |
RUN | hrrr_control_restart | - noaacloud | | hrrr_control
RUN | hrrr_control_restart_qr | - noaacloud | | hrrr_control_qr
RUN | rrfs_v1beta | | baseline |
RUN | rrfs_v1nssl | | baseline |
RUN | rrfs_v1nssl_nohailnoccn | | baseline |
Expand Down Expand Up @@ -192,13 +188,11 @@ COMPILE | rrfs_dyn32_phy32 | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D
RUN | regional_spp_sppt_shum_skeb_dyn32_phy32 | - noaacloud | baseline |
RUN | rap_control_dyn32_phy32 | - noaacloud | baseline |
RUN | hrrr_control_dyn32_phy32 | - noaacloud | baseline |
RUN | hrrr_control_qr_dyn32_phy32 | - noaacloud | |
RUN | rap_2threads_dyn32_phy32 | - noaacloud | |
RUN | hrrr_control_2threads_dyn32_phy32 | - noaacloud | |
RUN | hrrr_control_decomp_dyn32_phy32 | - noaacloud | |
RUN | rap_restart_dyn32_phy32 | - noaacloud | | rap_control_dyn32_phy32
RUN | hrrr_control_restart_dyn32_phy32 | - noaacloud | | hrrr_control_dyn32_phy32
RUN | hrrr_control_restart_qr_dyn32_phy32 | - noaacloud | | hrrr_control_qr_dyn32_phy32

COMPILE | rrfs_dyn32_phy32_faster | intel | -DAPP=ATM -DFASTER=ON -DCCPP_SUITES=FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON | - noaacloud | fv3 |
RUN | conus13km_control | - noaacloud | baseline |
Expand Down Expand Up @@ -234,15 +228,11 @@ RUN | hafs_regional_atm_ocn |
RUN | hafs_regional_atm_wav | | baseline |
RUN | hafs_regional_atm_ocn_wav | - noaacloud | baseline |
RUN | hafs_regional_1nest_atm | - jet s4 noaacloud | baseline |
RUN | hafs_regional_1nest_atm_qr | - jet s4 cheyenne noaacloud | |
RUN | hafs_regional_telescopic_2nests_atm | - jet s4 noaacloud | baseline |
RUN | hafs_global_1nest_atm | - jet s4 noaacloud | baseline |
RUN | hafs_global_1nest_atm_qr | - jet s4 cheyenne noaacloud | |
RUN | hafs_global_multiple_4nests_atm | - jet s4 noaacloud | baseline |
RUN | hafs_global_multiple_4nests_atm_qr | - jet s4 cheyenne noaacloud | |
RUN | hafs_regional_specified_moving_1nest_atm | - jet s4 noaacloud | baseline |
RUN | hafs_regional_storm_following_1nest_atm | - jet s4 noaacloud | baseline |
RUN | hafs_regional_storm_following_1nest_atm_qr | - jet s4 cheyenne noaacloud | |
RUN | hafs_regional_storm_following_1nest_atm_ocn | - jet s4 noaacloud | baseline |
RUN | hafs_global_storm_following_1nest_atm | - jet s4 noaacloud | baseline |

Expand Down Expand Up @@ -333,11 +323,11 @@ RUN | rap_sfcdiff | + hera cheyenne hercul
RUN | rap_sfcdiff_decomp | + hera cheyenne hercules | |
RUN | rap_sfcdiff_restart | + hera cheyenne hercules | | rap_sfcdiff
RUN | hrrr_control | + hera cheyenne hercules | baseline |
RUN | hrrr_control_qr | + hera cheyenne hercules | |
RUN | hrrr_control_noqr | + hera cheyenne hercules | |
RUN | hrrr_control_2threads | + hera cheyenne hercules | |
RUN | hrrr_control_decomp | + hera cheyenne hercules | |
RUN | hrrr_control_restart | + hera cheyenne hercules | | hrrr_control
RUN | hrrr_control_restart_qr | + hera cheyenne hercules | | hrrr_control_qr
RUN | hrrr_control_restart_noqr | + hera cheyenne hercules | | hrrr_control_noqr
RUN | rrfs_v1beta | + hera cheyenne hercules | baseline |

### CCPP DEBUG tests ###
Expand Down Expand Up @@ -368,13 +358,11 @@ RUN | control_wam_debug | + hera cheyenne
COMPILE | rrfs_dyn32_phy32 | gnu | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON | + hera cheyenne hercules | fv3 |
RUN | rap_control_dyn32_phy32 | + hera cheyenne hercules | baseline |
RUN | hrrr_control_dyn32_phy32 | + hera cheyenne hercules | baseline |
RUN | hrrr_control_qr_dyn32_phy32 | + hera cheyenne hercules | |
RUN | rap_2threads_dyn32_phy32 | + hera cheyenne hercules | |
RUN | hrrr_control_2threads_dyn32_phy32 | + hera cheyenne hercules | |
RUN | hrrr_control_decomp_dyn32_phy32 | + hera cheyenne hercules | |
RUN | rap_restart_dyn32_phy32 | + hera cheyenne hercules | | rap_control_dyn32_phy32
RUN | hrrr_control_restart_dyn32_phy32 | + hera cheyenne hercules | | hrrr_control_dyn32_phy32
RUN | hrrr_control_restart_qr_dyn32_phy32 | + hera cheyenne hercules | | hrrr_control_qr_dyn32_phy32
RUN | conus13km_control | + hera cheyenne hercules | baseline |
RUN | conus13km_2threads | + hera cheyenne hercules | | conus13km_control
RUN | conus13km_restart_mismatch | + hera cheyenne hercules | baseline | conus13km_control
Expand Down
2 changes: 1 addition & 1 deletion tests/tests/control_c384
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export atm_omp_num_threads=${THRD_c384}
export INPES=${INPES_c384}
export JNPES=${JNPES_c384}
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=48
export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP_c384:-48}
export FHMAX=12
export NPZ=127
export NPZP=128
Expand Down
5 changes: 4 additions & 1 deletion tests/tests/control_c384gdas
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ export atm_omp_num_threads=${THRD_c384}
export INPES=${INPES_c384}
export JNPES=${JNPES_c384}
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=48
export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP_c384gdas:-48}
export FHMAX=12
export NPZ=127
export NPZP=128
Expand Down Expand Up @@ -98,3 +98,6 @@ export INPUT_NML=control_gdas.nml.IN
if [[ $MACHINE_ID = gaea || $MACHINE_ID = cheyenne ]]; then
TPN=18
fi
if [[ $MACHINE_ID = jet ]] ; then
TPN=20
fi
2 changes: 1 addition & 1 deletion tests/tests/control_qr_p8 → tests/tests/control_noqr_p8
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ export SYEAR=2021
export SMONTH=03
export SDAY=22
export SHOUR=06
export QUILTING_RESTART=.true.
export QUILTING_RESTART=.false.
export RESTART_INTERVAL="12 -1"
export OUTPUT_GRID='gaussian_grid'
export WRITE_DOPOST=.true.
Expand Down
2 changes: 1 addition & 1 deletion tests/tests/control_restart_qr_p8 → tests/tests/control_restart_noqr_p8
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ export SDAY=22
export SHOUR=06
export OUTPUT_GRID='gaussian_grid'
export WRITE_DOPOST=.true.
export QUILTING_RESTART=.true.
export QUILTING_RESTART=.false.
export FHROT=12
export RESTART_FILE_PREFIX="${SYEAR}${SMONTH}${SDAY}.$(printf "%02d" $(( ${SHOUR} + ${FHROT} )))0000"

Expand Down
1 change: 1 addition & 0 deletions tests/tests/conus13km_2threads
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,4 @@ export atm_omp_num_threads=2
export INPES=12
export JNPES=12
export WRTTASK_PER_GROUP=6
export QUILTING_RESTART=.false.
1 change: 1 addition & 0 deletions tests/tests/conus13km_control
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ export LIST_FILES="sfcf000.nc \

export_hrrr_conus13km
export RESTART_INTERVAL=1
export QUILTING_RESTART=.false.

if [[ " hera orion hercules gaea jet " =~ " ${MACHINE_ID} " ]] ; then
ZSTANDARD_LEVEL=5
Expand Down
1 change: 1 addition & 0 deletions tests/tests/conus13km_debug
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,4 @@ export LIST_FILES="sfcf000.nc \
export_hrrr_conus13km
export RESTART_INTERVAL=1
export FHMAX=1
export QUILTING_RESTART=.false.
1 change: 1 addition & 0 deletions tests/tests/conus13km_debug_2threads
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ export atm_omp_num_threads=2
export INPES=12
export JNPES=12
export WRTTASK_PER_GROUP=6
export QUILTING_RESTART=.false.
1 change: 1 addition & 0 deletions tests/tests/conus13km_debug_decomp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,4 @@ export RESTART_INTERVAL=1
export FHMAX=1
export INPES=16
export JNPES=9
export QUILTING_RESTART=.false.
1 change: 0 additions & 1 deletion tests/tests/conus13km_debug_qr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,4 @@ export LIST_FILES="sfcf000.nc \

export_hrrr_conus13km
export RESTART_INTERVAL=1
export QUILTING_RESTART=.true.
export FHMAX=1
1 change: 1 addition & 0 deletions tests/tests/conus13km_decomp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,4 @@ export_hrrr_conus13km
export RESTART_INTERVAL=1
export INPES=16
export JNPES=9
export QUILTING_RESTART=.false.
1 change: 1 addition & 0 deletions tests/tests/conus13km_radar_tten_debug
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@ export_hrrr_conus13km
export RESTART_INTERVAL=1
export FH_DFI_RADAR='0.0, 0.25, 0.50, 0.75, 1.0'
export FHMAX=1
export QUILTING_RESTART=.false.
1 change: 1 addition & 0 deletions tests/tests/conus13km_restart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,4 @@ export_hrrr_conus13km
export FHROT=1
export RESTART_FILE_PREFIX="${SYEAR}${SMONTH}${SDAY}.$(printf "%02d" $(( ${SHOUR} + ${FHROT} )))0000"
export RRFS_RESTART=YES
export QUILTING_RESTART=.false.
1 change: 1 addition & 0 deletions tests/tests/conus13km_restart_mismatch
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,4 @@ export_hrrr_conus13km
export FHROT=1
export RESTART_FILE_PREFIX="${SYEAR}${SMONTH}${SDAY}.$(printf "%02d" $(( ${SHOUR} + ${FHROT} )))0000"
export RRFS_RESTART=YES
export QUILTING_RESTART=.false.
4 changes: 0 additions & 4 deletions tests/tests/cpld_control_p8
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,3 @@ export FV3_RUN=cpld_control_run.IN
if [[ $MACHINE_ID = cheyenne ]]; then
TPN=18
fi

if [[ $MACHINE_ID = hera ]] && [[ $RT_COMPILER = gnu ]]; then
WLCLK=40
fi
Loading

0 comments on commit 17643bc

Please sign in to comment.