From aada7736c51b5f486b8d9251933184d70d9bb494 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 6 Aug 2020 10:22:28 -0600 Subject: [PATCH] Make stochastic physics independent of fv3atm and its submodules (#145) * atmos_model.F90: remove dependency of stochastic physics on IPD_Control * Add mpi_wrapper to stochastic_physics to substitute calls to fv_mp_mod routines * Make cellular automata code independent of fv_mp_mod * Make cellular automata code independent of atmosphere_mod * Add halo exchange routine to stochastic_physics code * Remove dependency on GFS_coupling_type from stochastic_physics.F90 * Remove dependency on GFS_grid_type in stochastic_physics and stochastic_physics_sfc * Cleanup work, avoid unnecessary data copies * Move stochastic physics code into its own wrapper/driver * Remove dependency on Statein, Coupling and Intdiag DDT from cellular_automata_sgs.F90 * Compile kinddef module, cleanup stochastic_physics_wrapper.F90 * Update cmake build system * Fix a bug in atmos_model.F90, inadvertetly removed a closing parenthesis * Update to build system for stochastic_physics_wrapper following cmake updates * Update gnumake build to include stochastic_physics wrapper * Remove commented out lines from stochastic_physics/makefile Co-authored-by: Jun Wang --- .github/CODEOWNERS | 7 + .github/ISSUE_TEMPLATE/bug_report.md | 56 ++++ .github/ISSUE_TEMPLATE/feature_request.md | 32 +++ .github/pull_request_template.md | 37 +++ CMakeLists.txt | 3 +- atmos_model.F90 | 72 +---- makefile | 10 +- stochastic_physics/CMakeLists.txt | 69 +---- stochastic_physics/makefile | 54 ++++ .../stochastic_physics_wrapper.F90 | 249 ++++++++++++++++++ 10 files changed, 461 insertions(+), 128 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/pull_request_template.md create mode 100644 stochastic_physics/makefile create mode 100644 stochastic_physics/stochastic_physics_wrapper.F90 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..21c5ba4f4 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# This is a comment. +# Each line is a file pattern followed by one or more owners. +# Order is important; the last matching pattern takes the most +# precedence. + +# The components are owned by their respective teams +/FV3/ @noaa-emc/fv3atm-team diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..faa664ab3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,56 @@ +--- +name: Bug report +about: Create a report to fix bugs +title: '' +labels: bug +assignees: '' + +--- + +## Description +Provide a clear and concise description of what the bug is. +Also give a description of how to fix the bug. + + +### To Reproduce: +What compilers/machines are you seeing this with? +Give explicit steps to reproduce the behavior. +1. do this +2. then that +3. then, oops, look at the bug + + +## Additional context +Add any other context about the problem here. +Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example: +- needs to be fixed also in noaa-emc/nems/issues/ +- needed for noaa-emc/fv3atm/pull/ + + +## Output + +**Screenshots** +If applicable, drag and drop screenshots to help explain your problem. + +**output logs** +If applicable, include relevant output logs. +Either drag and drop the entire log file here (if a long log) or + +``` +paste the code here (if a short section of log) +``` + +## Testing: + +1. Have you tested the code changes? On what platforms? + +2. Have you run regression test in ufs-weather-model or ufs-s2s-model with code changes? +- Will the baseline results change? +- If the baseline results change, is it expected? Please give brief explanation. + +## Dependent PRs: + +Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example: +- required to support noaa-emc/GFDL_atmos_cubed_sphere/issues/ +- ncar/ccpp-physics/pull/ +- associated ufs-weather-model/pull/ diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..12db08eda --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,32 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +## Description +Provide a clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +## Solution +Add a clear and concise description of what solution you provide. + +## Alternatives +If applicable, add a description of any alternative solutions or features you've considered. + +## Testing: + +1. Have you tested the code changes? On what platforms? +2. Have you run regression test in ufs-weather-model or ufs-s2s-model with code changes? +- Will the baseline results change? +- If yes, please give brief explanation of your code updates on the regression test. + +## Dependent PRs: + +Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example: +- required to support noaa-emc/GFDL_atmos_cubed_sphere/issues/ +- ncar/ccpp-physics/pull/ +- associated ufs-weather-model/pull/ + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..e4d67029f --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,37 @@ +## Description + +(Instructions: this, and all subsequent sections of text should be removed and filled in as appropriate.) +Provide a detailed description of what this PR does. +What bug does it fix, or what feature does it add? +Is a change of answers expected from this PR? + + + +### Issue(s) addressed + +Link the issues to be closed with this PR, whether in this repository, or in another repository. +(Remember, issues should always be created before starting work on a PR branch!) +- fixes # +- fixes noaa-emc/fv3atm/issues/ + + + +## Testing + +How were these changes tested? +What compilers / HPCs was it tested with? +Are the changes covered by regression tests? (If not, why? Do new tests need to be added?) +Have the ufs-weather-model regression test been run? On what platform? +- Will the code updates change regression test baseline? If yes, why? Please show the baseline directory below. +- Please commit the regression test log files in your ufs-weather-model branch + + +## Dependencies + +If testing this branch requires non-default branches in other repositories, list them. +Those branches should have matching names (ideally) + +Do PRs in upstream repositories need to be merged first? +If so add the "waiting for other repos" label and list the upstream PRs +- waiting on noaa-emc/nems/pull/ +- waiting on noaa-emc/fv3atm/pull/ diff --git a/CMakeLists.txt b/CMakeLists.txt index d85272169..1eaae5928 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,6 +129,7 @@ target_link_libraries(fv3atm PUBLIC ipd) target_link_libraries(fv3atm PUBLIC io) target_link_libraries(fv3atm PUBLIC fv3dycore) target_link_libraries(fv3atm PUBLIC stochastic_physics) +target_link_libraries(fv3atm PUBLIC stochastic_physics_wrapper) target_link_libraries(fv3atm PUBLIC nemsio::nemsio w3emc::w3emc_d w3nco::w3nco_d sp::sp_d bacio::bacio_4) @@ -142,7 +143,7 @@ target_link_libraries(fv3atm PUBLIC esmf) ### Install ############################################################################### install( - TARGETS fv3atm fv3dycore io ipd gfsphysics ${CCPP_LIBRARIES} cpl stochastic_physics + TARGETS fv3atm fv3dycore io ipd gfsphysics ${CCPP_LIBRARIES} cpl stochastic_physics stochastic_physics_wrapper EXPORT fv3atm-config LIBRARY DESTINATION lib ARCHIVE DESTINATION lib) diff --git a/atmos_model.F90 b/atmos_model.F90 index 8637ed838..ec8e8a9a4 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -104,9 +104,7 @@ module atmos_model_mod use physics_abstraction_layer, only: time_vary_step, radiation_step1, physics_step1, physics_step2 #endif -use stochastic_physics, only: init_stochastic_physics, & - run_stochastic_physics -use stochastic_physics_sfc, only: run_stochastic_physics_sfc +use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper use FV3GFS_io_mod, only: FV3GFS_restart_read, FV3GFS_restart_write, & FV3GFS_IPD_checksum, & @@ -251,26 +249,17 @@ module atmos_model_mod ! subroutine update_atmos_radiation_physics (Atmos) -#ifdef OPENMP - use omp_lib -#endif !----------------------------------------------------------------------- type (atmos_data_type), intent(in) :: Atmos !--- local variables--- integer :: nb, jdat(8), rc procedure(IPD_func0d_proc), pointer :: Func0d => NULL() procedure(IPD_func1d_proc), pointer :: Func1d => NULL() - integer :: nthrds + ! #ifdef CCPP integer :: ierr #endif -#ifdef OPENMP - nthrds = omp_get_max_threads() -#else - nthrds = 1 -#endif - if (mpp_pe() == mpp_root_pe() .and. debug) write(6,*) "statein driver" !--- get atmospheric state from the dynamic core call set_atmosphere_pelist() @@ -307,24 +296,7 @@ subroutine update_atmos_radiation_physics (Atmos) #endif !--- call stochastic physics pattern generation / cellular automata - if (IPD_Control%do_sppt .OR. IPD_Control%do_shum .OR. IPD_Control%do_skeb .OR. IPD_Control%do_sfcperts) then - call run_stochastic_physics(IPD_Control, IPD_Data(:)%Grid, IPD_Data(:)%Coupling, nthrds) - end if - - if(IPD_Control%do_ca)then - if(IPD_Control%ca_sgs)then - call cellular_automata_sgs(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, & - IPD_Control%nca,IPD_Control%ncells,IPD_Control%nlives,IPD_Control%nfracseed,& - IPD_Control%nseed,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,& - IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1)) - endif - if(IPD_Control%ca_global)then - call cellular_automata_global(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, & - IPD_Control%nca_g,IPD_Control%ncells_g,IPD_Control%nlives_g,IPD_Control%nfracseed,& - IPD_Control%nseed_g,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,& - IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1),IPD_Control%nsmooth,IPD_Control%ca_amplitude) - endif - endif + call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block) !--- if coupled, assign coupled fields if( IPD_Control%cplflx .or. IPD_Control%cplwav ) then @@ -649,45 +621,11 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step) call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm) #endif - if (IPD_Control%do_sppt .OR. IPD_Control%do_shum .OR. IPD_Control%do_skeb .OR. IPD_Control%do_sfcperts) then - ! Initialize stochastic physics - call init_stochastic_physics(IPD_Control, Init_parm, mpp_npes(), nthrds) - if(IPD_Control%me == IPD_Control%master) print *,'do_skeb=',IPD_Control%do_skeb - end if +!--- Initialize stochastic physics pattern generation / cellular automata for first time step + call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block) Atmos%Diag => IPD_Diag - if (IPD_Control%do_sfcperts) then - ! Get land surface perturbations here (move to GFS_time_vary - ! step if wanting to update each time-step) - call run_stochastic_physics_sfc(IPD_Control, IPD_Data(:)%Grid, IPD_Data(:)%Coupling) - end if - - ! Initialize cellular automata - if(IPD_Control%do_ca)then - ! DH* The current implementation of cellular_automata assumes that all blocksizes are the - ! same - abort if this is not the case, otherwise proceed with Atm_block%blksz(1) below - if (.not. minval(Atm_block%blksz)==maxval(Atm_block%blksz)) then - call mpp_error(FATAL, 'Logic errror: cellular_automata not compatible with non-uniform blocksizes') - end if - ! *DH - if(IPD_Control%do_ca)then - if(IPD_Control%ca_sgs)then - call cellular_automata_sgs(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, & - IPD_Control%nca,IPD_Control%ncells,IPD_Control%nlives,IPD_Control%nfracseed,& - IPD_Control%nseed,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,& - IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1)) - endif - if(IPD_Control%ca_global)then - call cellular_automata_global(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, & - IPD_Control%nca_g,IPD_Control%ncells_g,IPD_Control%nlives_g,IPD_Control%nfracseed,& - IPD_Control%nseed_g,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,& - IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1),IPD_Control%nsmooth,IPD_Control%ca_amplitude) - endif - - endif - endif - Atm(mygrid)%flagstruct%do_skeb = IPD_Control%do_skeb ! initialize the IAU module diff --git a/makefile b/makefile index 7ea34a736..4191b4777 100644 --- a/makefile +++ b/makefile @@ -12,7 +12,7 @@ ifeq ($(strip $(exist)),) endif -FFLAGS += -I$(FMS_DIR) -Igfsphysics -Iipd -Icpl -Iio -Iatmos_cubed_sphere -Iccpp/driver -I../stochastic_physics +FFLAGS += -I$(FMS_DIR) -Igfsphysics -Iipd -Icpl -Iio -Iatmos_cubed_sphere -Iccpp/driver -Istochastic_physics CPPDEFS += -DESMF_VERSION_MAJOR=$(ESMF_VERSION_MAJOR) # Flag to CCPP build for 32bit dynamics @@ -41,8 +41,9 @@ libs: $(MAKE) -C io $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) $(MAKE) -C atmos_cubed_sphere $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) $(MAKE) -C ../stochastic_physics $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) 32BIT=N # force gfs physics to 64bit + $(MAKE) -C stochastic_physics $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) 32BIT=N # force gfs physics to 64bit -$(FV3_EXE): atmos_model.o coupler_main.o ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a ../stochastic_physics/libstochastic_physics.a cpl/libfv3cpl.a +$(FV3_EXE): atmos_model.o coupler_main.o ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a stochastic_physics/libstochastic_physics_wrapper.a ../stochastic_physics/libstochastic_physics.a cpl/libfv3cpl.a $(LD) -o $@ $^ $(NCEPLIBS) $(LDFLAGS) else @@ -87,7 +88,7 @@ endif ifneq (,$(findstring CCPP,$(CPPDEFS))) esmf_make_fragment: @rm -rf nems_dir; mkdir nems_dir - @cp $(FV3CAP_LIB) ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a cpl/libfv3cpl.a ../stochastic_physics/libstochastic_physics.a nems_dir + @cp $(FV3CAP_LIB) ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a cpl/libfv3cpl.a stochastic_physics/libstochastic_physics_wrapper.a ../stochastic_physics/libstochastic_physics.a nems_dir @cp fv3gfs_cap_mod.mod nems_dir @echo "# ESMF self-describing build dependency makefile fragment" > fv3.mk @echo "# src location $(PWD)" >> fv3.mk @@ -97,7 +98,7 @@ esmf_make_fragment: #@echo "ESMF_DEP_INCPATH = $(PWD)/nems_dir" >> fv3.mk @echo "ESMF_DEP_INCPATH = $(PWD) $(addprefix $(PWD)/, nems_dir ccpp/driver atmos_cubed_sphere io gfsphysics cpl ipd ../stochastic_physics)" >> fv3.mk @echo "ESMF_DEP_CMPL_OBJS =" >> fv3.mk - @echo "ESMF_DEP_LINK_OBJS = $(addprefix $(PWD)/nems_dir/, libfv3cap.a libccppdriver.a libfv3core.a libfv3io.a libipd.a libgfsphys.a libfv3cpl.a libstochastic_physics.a) $(SIONLIB_LINK_FLAGS)" >> fv3.mk + @echo "ESMF_DEP_LINK_OBJS = $(addprefix $(PWD)/nems_dir/, libfv3cap.a libccppdriver.a libfv3core.a libfv3io.a libipd.a libgfsphys.a libfv3cpl.a libstochastic_physics_wrapper.a libstochastic_physics.a) $(SIONLIB_LINK_FLAGS)" >> fv3.mk @echo "ESMF_DEP_SHRD_PATH =" >> fv3.mk @echo "ESMF_DEP_SHRD_LIBS =" >> fv3.mk @echo @@ -143,6 +144,7 @@ clean: (cd ccpp/driver && make clean) (cd ipd && make clean) (cd ../stochastic_physics && make clean) + (cd stochastic_physics && make clean) (cd io && make clean) (cd atmos_cubed_sphere && make clean) (cd cpl && make clean) diff --git a/stochastic_physics/CMakeLists.txt b/stochastic_physics/CMakeLists.txt index 4f4186464..c9131308b 100644 --- a/stochastic_physics/CMakeLists.txt +++ b/stochastic_physics/CMakeLists.txt @@ -1,62 +1,19 @@ - -if(32BIT) -remove_definitions(-DOVERLOAD_R4) -remove_definitions(-DOVERLOAD_R8) -message ("Force 64 bits in stochastic_physics") -if(CMAKE_Fortran_COMPILER_ID MATCHES "Intel") - if(REPRO) - string (REPLACE "-i4 -real-size 32" "-i4 -real-size 64" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") - else() - string (REPLACE "-i4 -real-size 32" "-i4 -real-size 64 -no-prec-div -no-prec-sqrt" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") - endif() -elseif(CMAKE_Fortran_COMPILER_ID MATCHES "GNU") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8") -endif() -endif() - add_library( - stochastic_physics + stochastic_physics_wrapper - ../../stochastic_physics/plumes.f90 - - ../../stochastic_physics/stochy_gg_def.f - ../../stochastic_physics/stochy_resol_def.f - ../../stochastic_physics/stochy_layout_lag.f - ../../stochastic_physics/four_to_grid_stochy.F - ../../stochastic_physics/glats_stochy.f - ../../stochastic_physics/sumfln_stochy.f - ../../stochastic_physics/gozrineo_stochy.f - ../../stochastic_physics/num_parthds_stochy.f - ../../stochastic_physics/get_ls_node_stochy.f - ../../stochastic_physics/get_lats_node_a_stochy.f - ../../stochastic_physics/setlats_a_stochy.f - ../../stochastic_physics/setlats_lag_stochy.f - ../../stochastic_physics/epslon_stochy.f - ../../stochastic_physics/getcon_lag_stochy.f - ../../stochastic_physics/pln2eo_stochy.f - ../../stochastic_physics/dozeuv_stochy.f - ../../stochastic_physics/dezouv_stochy.f - - ../../stochastic_physics/spectral_layout.F90 - ../../stochastic_physics/getcon_spectral.F90 - ../../stochastic_physics/stochy_namelist_def.F90 - ../../stochastic_physics/compns_stochy.F90 - ../../stochastic_physics/stochy_internal_state_mod.F90 - ../../stochastic_physics/stochastic_physics.F90 - ../../stochastic_physics/stochy_patterngenerator.F90 - ../../stochastic_physics/stochy_data_mod.F90 - ../../stochastic_physics/get_stochy_pattern.F90 - ../../stochastic_physics/initialize_spectral_mod.F90 - ../../stochastic_physics/cellular_automata_global.F90 - ../../stochastic_physics/cellular_automata_sgs.F90 - ../../stochastic_physics/update_ca.F90 + ./stochastic_physics_wrapper.F90 ) +target_include_directories(stochastic_physics_wrapper PRIVATE ${CMAKE_BINARY_DIR}/stochastic_physics) if(CCPP) - target_include_directories(stochastic_physics PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) - target_include_directories(stochastic_physics PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/physics) + target_include_directories(stochastic_physics_wrapper PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) + target_include_directories(stochastic_physics_wrapper PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/physics) endif() -target_link_libraries(stochastic_physics PRIVATE fms) -target_link_libraries(stochastic_physics PRIVATE gfsphysics) -target_link_libraries(stochastic_physics PRIVATE fv3dycore) -target_link_libraries(stochastic_physics PRIVATE sp::sp_d) +# DH* is there a better way to do this? +target_include_directories(stochastic_physics_wrapper PUBLIC $) +# *DH +target_link_libraries(stochastic_physics_wrapper fms) +target_link_libraries(stochastic_physics_wrapper stochastic_physics) +target_link_libraries(stochastic_physics_wrapper gfsphysics) +target_link_libraries(stochastic_physics_wrapper fv3dycore) + diff --git a/stochastic_physics/makefile b/stochastic_physics/makefile new file mode 100644 index 000000000..eb721c8c4 --- /dev/null +++ b/stochastic_physics/makefile @@ -0,0 +1,54 @@ +SHELL = /bin/sh + +inside_nems := $(wildcard ../../../conf/configure.nems) +ifneq ($(strip $(inside_nems)),) + include ../../../conf/configure.nems +else + exist_configure_fv3 := $(wildcard ../conf/configure.fv3) + ifneq ($(strip $(exist_configure_fv3)),) + include ../conf/configure.fv3 + else + $(error "../conf/configure.fv3 file is missing. Run ./configure") + endif + $(info ) + $(info Build standalone FV3 io ...) + $(info ) +endif +$(info $$ESMF_INC is [${ESMF_INC}]) + +LIBRARY = libstochastic_physics_wrapper.a + +FFLAGS += -I$(FMS_DIR) -I ../../stochastic_physics -I../ccpp/physics -I../atmos_cubed_sphere + +SRCS_f = + +SRCS_f90 = + +SRCS_F = + +SRCS_F90 = stochastic_physics_wrapper.F90 + +SRCS_c = + +DEPEND_FILES = $(SRCS_f) $(SRCS_f90) $(SRCS_F) $(SRCS_F90) + +OBJS_f = $(SRCS_f:.f=.o) +OBJS_f90 = $(SRCS_f90:.f90=.o) +OBJS_F = $(SRCS_F:.F=.o) +OBJS_F90 = $(SRCS_F90:.F90=.o) +OBJS_c = $(SRCS_c:.c=.o) + +OBJS = $(OBJS_f) $(OBJS_f90) $(OBJS_F) $(OBJS_F90) $(OBJS_c) + +all default: depend $(LIBRARY) + +$(LIBRARY): $(OBJS) + $(AR) $(ARFLAGS) $@ $? + +.PHONY: clean +clean: + @echo "Cleaning io ... " + @echo + $(RM) -f $(LIBRARY) *__genmod.f90 *.o *.mod *.i90 *.lst *.i + +include ../conf/make.rules diff --git a/stochastic_physics/stochastic_physics_wrapper.F90 b/stochastic_physics/stochastic_physics_wrapper.F90 new file mode 100644 index 000000000..2fb8cafd1 --- /dev/null +++ b/stochastic_physics/stochastic_physics_wrapper.F90 @@ -0,0 +1,249 @@ +module stochastic_physics_wrapper_mod + + use machine, only: kind_phys + + implicit none + + ! For stochastic physics pattern generation + real(kind=kind_phys), dimension(:,:), allocatable, save :: xlat + real(kind=kind_phys), dimension(:,:), allocatable, save :: xlon + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: sppt_wts + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: shum_wts + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: skebu_wts + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: skebv_wts + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: sfc_wts + + ! For cellular automata + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: ugrs + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: qgrs + real(kind=kind_phys), dimension(:,:), allocatable, save :: pgr + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: vvl + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: prsl + real(kind=kind_phys), dimension(:,:), allocatable, save :: condition + real(kind=kind_phys), dimension(:,:), allocatable, save :: ca_deep_cpl, ca_turb_cpl, ca_shal_cpl + real(kind=kind_phys), dimension(:,:), allocatable, save :: ca_deep_diag,ca_turb_diag,ca_shal_diag + real(kind=kind_phys), dimension(:,:), allocatable, save :: ca1_cpl, ca2_cpl, ca3_cpl + real(kind=kind_phys), dimension(:,:), allocatable, save :: ca1_diag,ca2_diag,ca3_diag + + +!---------------- +! Public Entities +!---------------- +! functions + public stochastic_physics_wrapper + + contains + +!------------------------------- +! CCPP step +!------------------------------- + subroutine stochastic_physics_wrapper (GFS_Control, GFS_Data, Atm_block) + +#ifdef OPENMP + use omp_lib +#endif + + use GFS_typedefs, only: GFS_control_type, GFS_data_type + use mpp_mod, only: FATAL, mpp_error + use block_control_mod, only: block_control_type + use atmosphere_mod, only: Atm, mygrid + + use stochastic_physics, only: init_stochastic_physics, run_stochastic_physics + use stochastic_physics_sfc, only: run_stochastic_physics_sfc + use cellular_automata_global_mod, only: cellular_automata_global + use cellular_automata_sgs_mod, only: cellular_automata_sgs + + implicit none + + type(GFS_control_type), intent(inout) :: GFS_Control + type(GFS_data_type), intent(inout) :: GFS_Data(:) + type(block_control_type), intent(inout) :: Atm_block + + integer :: nthreads, nb + +#ifdef OPENMP + nthreads = omp_get_max_threads() +#else + nthreads = 1 +#endif + + ! Initialize + initalize_stochastic_physics: if (GFS_Control%kdt==0) then + + if (GFS_Control%do_sppt .OR. GFS_Control%do_shum .OR. GFS_Control%do_skeb .OR. GFS_Control%do_sfcperts) then + ! Initialize stochastic physics + call init_stochastic_physics(GFS_Control%levs, GFS_Control%blksz, GFS_Control%dtp, & + GFS_Control%input_nml_file, GFS_Control%fn_nml, GFS_Control%nlunit, GFS_Control%do_sppt, GFS_Control%do_shum, & + GFS_Control%do_skeb, GFS_Control%do_sfcperts, GFS_Control%use_zmtnblck, GFS_Control%skeb_npass, GFS_Control%nsfcpert, & + GFS_Control%pertz0, GFS_Control%pertzt, GFS_Control%pertshc, GFS_Control%pertlai, GFS_Control%pertalb, GFS_Control%pertvegf, & + GFS_Control%ak, GFS_Control%bk, nthreads, GFS_Control%master, GFS_Control%communicator) + end if + + ! Get land surface perturbations here (move to "else" block below if wanting to update each time-step) + if (GFS_Control%do_sfcperts) then + ! Copy blocked data into contiguous arrays; no need to copy sfc_wts in (intent out) + allocate(xlat(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(xlon(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(sfc_wts(1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + do nb=1,Atm_block%nblks + xlat(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Grid%xlat(:) + xlon(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Grid%xlon(:) + end do + call run_stochastic_physics_sfc(GFS_Control%blksz, xlat=xlat, xlon=xlon, sfc_wts=sfc_wts) + ! Copy contiguous data back; no need to copy xlat/xlon, these are intent(in) - just deallocate + do nb=1,Atm_block%nblks + GFS_Data(nb)%Coupling%sfc_wts(:,:) = sfc_wts(nb,1:GFS_Control%blksz(nb),:) + end do + deallocate(xlat) + deallocate(xlon) + deallocate(sfc_wts) + end if + + ! Consistency check for cellular automata + if(GFS_Control%do_ca)then + ! DH* The current implementation of cellular_automata assumes that all blocksizes are the + ! same - abort if this is not the case, otherwise proceed with Atm_block%blksz(1) below + if (.not. minval(Atm_block%blksz)==maxval(Atm_block%blksz)) then + call mpp_error(FATAL, 'Logic errror: cellular_automata not compatible with non-uniform blocksizes') + end if + ! *DH + endif + + else initalize_stochastic_physics + + if (GFS_Control%do_sppt .OR. GFS_Control%do_shum .OR. GFS_Control%do_skeb) then + ! Copy blocked data into contiguous arrays; no need to copy weights in (intent(out)) + allocate(xlat(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(xlon(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + do nb=1,Atm_block%nblks + xlat(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Grid%xlat(:) + xlon(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Grid%xlon(:) + end do + if (GFS_Control%do_sppt) then + allocate(sppt_wts(1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + end if + if (GFS_Control%do_shum) then + allocate(shum_wts(1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + end if + if (GFS_Control%do_skeb) then + allocate(skebu_wts(1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + allocate(skebv_wts(1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + end if + call run_stochastic_physics(GFS_Control%levs, GFS_Control%kdt, GFS_Control%phour, GFS_Control%blksz, xlat=xlat, xlon=xlon, & + sppt_wts=sppt_wts, shum_wts=shum_wts, skebu_wts=skebu_wts, skebv_wts=skebv_wts, nthreads=nthreads) + ! Copy contiguous data back; no need to copy xlat/xlon, these are intent(in) - just deallocate + deallocate(xlat) + deallocate(xlon) + if (GFS_Control%do_sppt) then + do nb=1,Atm_block%nblks + GFS_Data(nb)%Coupling%sppt_wts(:,:) = sppt_wts(nb,1:GFS_Control%blksz(nb),:) + end do + deallocate(sppt_wts) + end if + if (GFS_Control%do_shum) then + do nb=1,Atm_block%nblks + GFS_Data(nb)%Coupling%shum_wts(:,:) = shum_wts(nb,1:GFS_Control%blksz(nb),:) + end do + deallocate(shum_wts) + end if + if (GFS_Control%do_skeb) then + do nb=1,Atm_block%nblks + GFS_Data(nb)%Coupling%skebu_wts(:,:) = skebu_wts(nb,1:GFS_Control%blksz(nb),:) + GFS_Data(nb)%Coupling%skebv_wts(:,:) = skebv_wts(nb,1:GFS_Control%blksz(nb),:) + end do + deallocate(skebu_wts) + deallocate(skebv_wts) + end if + end if + + endif initalize_stochastic_physics + + ! Cellular automata code is identical for initialization (kstep=0) and time integration (kstep>0) + if(GFS_Control%do_ca)then + if(GFS_Control%ca_sgs)then + ! Allocate contiguous arrays; copy in as needed + allocate(ugrs (1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + allocate(qgrs (1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + allocate(pgr (1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(vvl (1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + allocate(prsl (1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + allocate(ca_deep_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(ca_turb_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(ca_shal_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(condition (1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(ca_deep_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(ca_turb_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(ca_shal_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + do nb=1,Atm_block%nblks + ugrs (nb,1:GFS_Control%blksz(nb),:) = GFS_Data(nb)%Statein%ugrs(:,:) + qgrs (nb,1:GFS_Control%blksz(nb),:) = GFS_Data(nb)%Statein%qgrs(:,:,1) + pgr (nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Statein%pgr(:) + vvl (nb,1:GFS_Control%blksz(nb),:) = GFS_Data(nb)%Statein%vvl(:,:) + prsl (nb,1:GFS_Control%blksz(nb),:) = GFS_Data(nb)%Statein%prsl(:,:) + condition (nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Coupling%condition(:) + ca_deep_cpl(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Coupling%ca_deep(:) + ca_turb_cpl(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Coupling%ca_turb(:) + ca_shal_cpl(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Coupling%ca_shal(:) + enddo + call cellular_automata_sgs(GFS_Control%kdt,ugrs,qgrs,pgr,vvl,prsl,condition,ca_deep_cpl,ca_turb_cpl,ca_shal_cpl, & + ca_deep_diag,ca_turb_diag,ca_shal_diag,Atm(mygrid)%domain_for_coupler,Atm_block%nblks, & + Atm_block%isc,Atm_block%iec,Atm_block%jsc,Atm_block%jec,Atm(mygrid)%npx,Atm(mygrid)%npy, GFS_Control%levs, & + GFS_Control%nca,GFS_Control%ncells,GFS_Control%nlives,GFS_Control%nfracseed, & + GFS_Control%nseed,GFS_Control%nthresh,GFS_Control%ca_global,GFS_Control%ca_sgs,GFS_Control%iseed_ca, & + GFS_Control%ca_smooth,GFS_Control%nspinup,Atm_block%blksz(1),GFS_Control%master,GFS_Control%communicator) + ! Copy contiguous data back as needed + do nb=1,Atm_block%nblks + GFS_Data(nb)%Intdiag%ca_deep(:) = ca_deep_diag(nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Intdiag%ca_turb(:) = ca_turb_diag(nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Intdiag%ca_shal(:) = ca_shal_diag(nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Coupling%ca_deep(:) = ca_deep_cpl (nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Coupling%ca_turb(:) = ca_turb_cpl (nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Coupling%ca_shal(:) = ca_shal_cpl (nb,1:GFS_Control%blksz(nb)) + enddo + deallocate(ugrs ) + deallocate(qgrs ) + deallocate(pgr ) + deallocate(vvl ) + deallocate(prsl ) + deallocate(condition ) + deallocate(ca_deep_cpl ) + deallocate(ca_turb_cpl ) + deallocate(ca_shal_cpl ) + deallocate(ca_deep_diag) + deallocate(ca_turb_diag) + deallocate(ca_shal_diag) + endif + if(GFS_Control%ca_global)then + ! Allocate contiguous arrays; no need to copy in (intent out) + allocate(ca1_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(ca2_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(ca3_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(ca1_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(ca2_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(ca3_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + call cellular_automata_global(GFS_Control%kdt,ca1_cpl,ca2_cpl,ca3_cpl,ca1_diag,ca2_diag,ca3_diag,Atm(mygrid)%domain_for_coupler, & + Atm_block%nblks,Atm_block%isc,Atm_block%iec,Atm_block%jsc,Atm_block%jec,Atm(mygrid)%npx,Atm(mygrid)%npy,GFS_Control%levs, & + GFS_Control%nca_g,GFS_Control%ncells_g,GFS_Control%nlives_g,GFS_Control%nfracseed,GFS_Control%nseed_g,GFS_Control%nthresh, & + GFS_Control%ca_global,GFS_Control%ca_sgs,GFS_Control%iseed_ca,GFS_Control%ca_smooth,GFS_Control%nspinup,Atm_block%blksz(1), & + GFS_Control%nsmooth,GFS_Control%ca_amplitude,GFS_Control%master,GFS_Control%communicator) + ! Copy contiguous data back + do nb=1,Atm_block%nblks + GFS_Data(nb)%Coupling%ca1(:) = ca1_cpl (nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Coupling%ca2(:) = ca2_cpl (nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Coupling%ca3(:) = ca3_cpl (nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Intdiag%ca1(:) = ca1_diag(nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Intdiag%ca2(:) = ca2_diag(nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Intdiag%ca3(:) = ca3_diag(nb,1:GFS_Control%blksz(nb)) + enddo + deallocate(ca1_cpl ) + deallocate(ca2_cpl ) + deallocate(ca3_cpl ) + deallocate(ca1_diag) + deallocate(ca2_diag) + deallocate(ca3_diag) + endif + endif + + end subroutine stochastic_physics_wrapper + +end module stochastic_physics_wrapper_mod