diff --git a/config_src/solo_driver/MOM_driver.F90 b/config_src/solo_driver/MOM_driver.F90 index c9383a4287..584282f27f 100644 --- a/config_src/solo_driver/MOM_driver.F90 +++ b/config_src/solo_driver/MOM_driver.F90 @@ -33,15 +33,20 @@ program MOM_main use MOM, only : get_MOM_state_elements, MOM_state_is_synchronized use MOM, only : step_offline use MOM_coms, only : Set_PElist - use MOM_domains, only : MOM_infra_init, MOM_infra_end + use MOM_domains, only : MOM_infra_init, MOM_infra_end, set_MOM_thread_affinity + use MOM_ensemble_manager, only : ensemble_manager_init, get_ensemble_size + use MOM_ensemble_manager, only : ensemble_pelist_setup use MOM_error_handler, only : MOM_error, MOM_mesg, WARNING, FATAL, is_root_pe use MOM_error_handler, only : callTree_enter, callTree_leave, callTree_waypoint use MOM_file_parser, only : read_param, get_param, log_param, log_version, param_file_type use MOM_file_parser, only : close_param_file use MOM_forcing_type, only : forcing, mech_forcing, forcing_diagnostics use MOM_forcing_type, only : mech_forcing_diags, MOM_forcing_chksum, MOM_mech_forcing_chksum - use MOM_get_input, only : directories + use MOM_get_input, only : get_MOM_input, directories use MOM_grid, only : ocean_grid_type + use MOM_ice_shelf, only : initialize_ice_shelf, ice_shelf_end, ice_shelf_CS + use MOM_ice_shelf, only : shelf_calc_flux, add_shelf_forces, ice_shelf_save_restart + use MOM_ice_shelf, only : initialize_ice_shelf_fluxes, initialize_ice_shelf_forces use MOM_interpolate, only : time_interp_external_init use MOM_io, only : file_exists, open_file, close_file use MOM_io, only : check_nml_error, io_infra_init, io_infra_end @@ -50,30 +55,19 @@ program MOM_main use MOM_string_functions,only : uppercase use MOM_surface_forcing, only : set_forcing, forcing_save_restart use MOM_surface_forcing, only : surface_forcing_init, surface_forcing_CS - use MOM_time_manager, only : time_type, set_date, get_date - use MOM_time_manager, only : real_to_time, time_type_to_real + use MOM_time_manager, only : time_type, set_date, get_date, real_to_time, time_type_to_real use MOM_time_manager, only : operator(+), operator(-), operator(*), operator(/) use MOM_time_manager, only : operator(>), operator(<), operator(>=) use MOM_time_manager, only : increment_date, set_calendar_type, month_name - use MOM_time_manager, only : JULIAN, GREGORIAN, NOLEAP, THIRTY_DAY_MONTHS - use MOM_time_manager, only : NO_CALENDAR + use MOM_time_manager, only : JULIAN, GREGORIAN, NOLEAP, THIRTY_DAY_MONTHS, NO_CALENDAR use MOM_tracer_flow_control, only : tracer_flow_control_CS use MOM_unit_scaling, only : unit_scale_type use MOM_variables, only : surface use MOM_verticalGrid, only : verticalGrid_type + use MOM_wave_interface, only : wave_parameters_CS, MOM_wave_interface_init + use MOM_wave_interface, only : MOM_wave_interface_init_lite, Update_Surface_Waves use MOM_write_cputime, only : write_cputime, MOM_write_cputime_init use MOM_write_cputime, only : write_cputime_start_clock, write_cputime_CS - use MOM_get_input, only : get_MOM_input - use ensemble_manager_mod, only : ensemble_manager_init, get_ensemble_size - use ensemble_manager_mod, only : ensemble_pelist_setup - use fms_affinity_mod, only : fms_affinity_init, fms_affinity_set,fms_affinity_get - - use MOM_ice_shelf, only : initialize_ice_shelf, ice_shelf_end, ice_shelf_CS - use MOM_ice_shelf, only : shelf_calc_flux, add_shelf_forces, ice_shelf_save_restart - use MOM_ice_shelf, only : initialize_ice_shelf_fluxes, initialize_ice_shelf_forces - - use MOM_wave_interface, only: wave_parameters_CS, MOM_wave_interface_init - use MOM_wave_interface, only: MOM_wave_interface_init_lite, Update_Surface_Waves implicit none @@ -253,13 +247,8 @@ program MOM_main endif endif -!$ call fms_affinity_init -!$ call fms_affinity_set('OCEAN', use_hyper_thread, ocean_nthreads) -!$ call omp_set_num_threads(ocean_nthreads) -!$OMP PARALLEL -!$ write(6,*) "ocean_solo OMPthreading ", fms_affinity_get(), omp_get_thread_num(), omp_get_num_threads() -!$ flush(6) -!$OMP END PARALLEL + ! This call sets the number and affinity of threads with openMP. + !$ call set_MOM_thread_affinity(ocean_nthreads, use_hyper_thread) ! Read ocean_solo restart, which can override settings from the namelist. if (file_exists(trim(dirs%restart_input_dir)//'ocean_solo.res')) then @@ -337,7 +326,7 @@ program MOM_main call callTree_waypoint("done surface_forcing_init") - call get_param(param_file,mod_name,"USE_WAVES",Use_Waves,& + call get_param(param_file,mod_name, "USE_WAVES", Use_Waves, & "If true, enables surface wave modules.",default=.false.) if (use_waves) then call MOM_wave_interface_init(Time, grid, GV, US, param_file, Waves_CSp, diag) diff --git a/src/framework/MOM_domain_infra.F90 b/src/framework/MOM_domain_infra.F90 index 5ced2e33c0..1f0594ef0d 100644 --- a/src/framework/MOM_domain_infra.F90 +++ b/src/framework/MOM_domain_infra.F90 @@ -498,7 +498,7 @@ subroutine pass_var_complete_3d(id_update, array, MOM_dom, sideflag, position, h end subroutine pass_var_complete_3d !> pass_vector_2d does a halo update for a pair of two-dimensional arrays -!! representing the compontents of a two-dimensional horizontal vector. +!! representing the components of a two-dimensional horizontal vector. subroutine pass_vector_2d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, complete, halo, & clock) real, dimension(:,:), intent(inout) :: u_cmpt !< The nominal zonal (u) component of the vector @@ -557,7 +557,7 @@ end subroutine pass_vector_2d !> fill_vector_symmetric_edges_2d does an usual set of halo updates that only !! fill in the values at the edge of a pair of symmetric memory two-dimensional -!! arrays representing the compontents of a two-dimensional horizontal vector. +!! arrays representing the components of a two-dimensional horizontal vector. !! If symmetric memory is not being used, this subroutine does nothing except to !! possibly turn optional cpu clocks on or off. subroutine fill_vector_symmetric_edges_2d(u_cmpt, v_cmpt, MOM_dom, stagger, scalar, & @@ -644,7 +644,7 @@ subroutine fill_vector_symmetric_edges_2d(u_cmpt, v_cmpt, MOM_dom, stagger, scal end subroutine fill_vector_symmetric_edges_2d !> pass_vector_3d does a halo update for a pair of three-dimensional arrays -!! representing the compontents of a three-dimensional horizontal vector. +!! representing the components of a three-dimensional horizontal vector. subroutine pass_vector_3d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, complete, halo, & clock) real, dimension(:,:,:), intent(inout) :: u_cmpt !< The nominal zonal (u) component of the vector @@ -702,7 +702,7 @@ subroutine pass_vector_3d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, complete, end subroutine pass_vector_3d !> pass_vector_start_2d starts a halo update for a pair of two-dimensional arrays -!! representing the compontents of a two-dimensional horizontal vector. +!! representing the components of a two-dimensional horizontal vector. function pass_vector_start_2d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, complete, halo, & clock) real, dimension(:,:), intent(inout) :: u_cmpt !< The nominal zonal (u) component of the vector @@ -759,7 +759,7 @@ function pass_vector_start_2d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, compl end function pass_vector_start_2d !> pass_vector_start_3d starts a halo update for a pair of three-dimensional arrays -!! representing the compontents of a three-dimensional horizontal vector. +!! representing the components of a three-dimensional horizontal vector. function pass_vector_start_3d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, complete, halo, & clock) real, dimension(:,:,:), intent(inout) :: u_cmpt !< The nominal zonal (u) component of the vector @@ -815,7 +815,7 @@ function pass_vector_start_3d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, compl end function pass_vector_start_3d !> pass_vector_complete_2d completes a halo update for a pair of two-dimensional arrays -!! representing the compontents of a two-dimensional horizontal vector. +!! representing the components of a two-dimensional horizontal vector. subroutine pass_vector_complete_2d(id_update, u_cmpt, v_cmpt, MOM_dom, direction, stagger, halo, & clock) integer, intent(in) :: id_update !< The integer id of this update which has been @@ -869,7 +869,7 @@ subroutine pass_vector_complete_2d(id_update, u_cmpt, v_cmpt, MOM_dom, direction end subroutine pass_vector_complete_2d !> pass_vector_complete_3d completes a halo update for a pair of three-dimensional -!! arrays representing the compontents of a three-dimensional horizontal vector. +!! arrays representing the components of a three-dimensional horizontal vector. subroutine pass_vector_complete_3d(id_update, u_cmpt, v_cmpt, MOM_dom, direction, stagger, halo, & clock) integer, intent(in) :: id_update !< The integer id of this update which has been @@ -1371,7 +1371,7 @@ function MOM_thread_affinity_set() !$ MOM_thread_affinity_set = (ocean_nthreads > 1 ) end function MOM_thread_affinity_set -!> set_MOM_thread_affinity sest the number of openMP threads to use with the ocean. +!> set_MOM_thread_affinity sets the number of openMP threads to use with the ocean. subroutine set_MOM_thread_affinity(ocean_nthreads, ocean_hyper_thread) integer, intent(in) :: ocean_nthreads !< Number of openMP threads to use for the ocean model logical, intent(in) :: ocean_hyper_thread !< If true, use hyper threading @@ -1379,10 +1379,13 @@ subroutine set_MOM_thread_affinity(ocean_nthreads, ocean_hyper_thread) ! Local variables !$ integer :: omp_get_thread_num, omp_get_num_threads !< These are the results of openMP functions + !$ call fms_affinity_init() ! fms_affinity_init can be safely called more than once. !$ call fms_affinity_set('OCEAN', ocean_hyper_thread, ocean_nthreads) !$ call omp_set_num_threads(ocean_nthreads) + !$OMP PARALLEL !$ write(6,*) "MOM_domains_mod OMPthreading ", fms_affinity_get(), omp_get_thread_num(), omp_get_num_threads() !$ flush(6) + !$OMP END PARALLEL end subroutine set_MOM_thread_affinity !> This subroutine retrieves the 1-d domains that make up the 2d-domain in a MOM_domain diff --git a/src/framework/MOM_domains.F90 b/src/framework/MOM_domains.F90 index 9ccef2888e..d230ecdf74 100644 --- a/src/framework/MOM_domains.F90 +++ b/src/framework/MOM_domains.F90 @@ -32,6 +32,7 @@ module MOM_domains ! Domain types and creation and destruction routines public :: MOM_domain_type, domain2D, domain1D public :: MOM_domains_init, create_MOM_domain, clone_MOM_domain, deallocate_MOM_domain +public :: MOM_thread_affinity_set, set_MOM_thread_affinity ! Domain query routines public :: get_domain_extent, get_domain_components, compute_block_extent, get_global_shape public :: PE_here, root_PE, num_PEs @@ -52,7 +53,7 @@ module MOM_domains contains !> MOM_domains_init initializes a MOM_domain_type variable, based on the information -!! read in from a param_file_type, and optionally returns data describing various' +!! read in from a param_file_type, and optionally returns data describing various !! properties of the domain type. subroutine MOM_domains_init(MOM_dom, param_file, symmetric, static_memory, & NIHALO, NJHALO, NIGLOBAL, NJGLOBAL, NIPROC, NJPROC, & @@ -260,7 +261,7 @@ subroutine MOM_domains_init(MOM_dom, param_file, symmetric, static_memory, & min_halo(1) = n_halo(1) n_halo(2) = max(n_halo(2), min_halo(2)) min_halo(2) = n_halo(2) - ! These are generally used only with static memory, so they are considerd layout params. + ! These are generally used only with static memory, so they are considered layout params. call log_param(param_file, mdl, "!NIHALO min_halo", n_halo(1), layoutParam=.true.) call log_param(param_file, mdl, "!NJHALO min_halo", n_halo(2), layoutParam=.true.) endif diff --git a/src/framework/MOM_ensemble_manager.F90 b/src/framework/MOM_ensemble_manager.F90 index 191dd79c9a..df1c30fc74 100644 --- a/src/framework/MOM_ensemble_manager.F90 +++ b/src/framework/MOM_ensemble_manager.F90 @@ -3,12 +3,16 @@ module MOM_ensemble_manager ! This file is part of MOM6. See LICENSE.md for the license. +use ensemble_manager_mod, only : ensemble_manager_init, ensemble_pelist_setup use ensemble_manager_mod, only : get_ensemble_id, get_ensemble_size use ensemble_manager_mod, only : get_ensemble_pelist, get_ensemble_filter_pelist implicit none ; private -public get_ensemble_id, get_ensemble_size, get_ensemble_pelist, get_ensemble_filter_pelist +public :: ensemble_manager_init, ensemble_pelist_setup +public :: get_ensemble_id, get_ensemble_size +public :: get_ensemble_pelist, get_ensemble_filter_pelist +! There need to be documented APIs in this module. end module MOM_ensemble_manager