diff --git a/.github/build.oasis3-mct.ubuntu22.04 b/.github/build.oasis3-mct.ubuntu22.04 new file mode 100644 index 0000000..8c7319f --- /dev/null +++ b/.github/build.oasis3-mct.ubuntu22.04 @@ -0,0 +1,81 @@ +# +# Include file for OASIS3 Makefile for a Linux system using +# GNU and OPENMPI. This file is used by the eCLM CI Test +# to build the OASIS3-MCT library. +# +# This file is based from: +# https://gitlab.com/cerfacs/oasis3-mct/-/blob/OASIS3-MCT_5.0/util/make_dir/header_examples/make.gfortran_openmpi_linux_openmp +# +############################################################################### +# +# CHAN : communication technique used in OASIS3 (MPI1/MPI2) +CHAN = MPI1 +# +# Paths for libraries, object files and binaries +# +# COUPLE : path for oasis3 main directory +COUPLE = $(OASIS_ROOT) +# +# BUILD_DIR : dir where the TopMakefileOasis3 is +BUILD_DIR = $(OASIS_ROOT)/util/make_dir +# +# +# ARCHDIR : directory created when compiling +ARCHDIR = $(HOME)/.local +# +# MPI library ((see the file /etc/modulefiles/mpi/openmpi-x86_64) +MPIDIR = /usr/lib/x86_64-linux-gnu/openmpi +MPIBIN = /usr/bin +MPI_INCLUDE = $(MPIDIR)/include +MPILIB = -L$(MPIDIR)/lib -lmpi +# +# NETCDF library of the system +NETCDF_INCLUDE1 = /usr/include +# netcdf.mod and hdf5.mod +NETCDF_INCLUDE2 = /usr/include +NETCDF_LIBRARY = -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf +# +# Compiling and other commands +MAKE = make +F90 = $(MPIBIN)/mpif90 -I$(MPI_INCLUDE) +F = $(F90) +f90 = $(F90) +f = $(F90) +CC = $(MPIBIN)/mpicc -I$(MPI_INCLUDE) +LD = $(MPIBIN)/mpif90 $(MPILIB) +AR = ar +ARFLAGS = -ruv +# +# CPP keys and compiler options +# +CPPDEF = -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY +# +# +#F90FLAGS_1 = -g -ffree-line-length-0 -fbounds-check -fopenmp +F90FLAGS_1 = -ffree-line-length-none -fallow-argument-mismatch -fopenmp +f90FLAGS_1 = $(F90FLAGS_1) +FFLAGS_1 = $(F90FLAGS_1) +fFLAGS_1 = $(F90FLAGS_1) +CCFLAGS_1 = +LDFLAGS = -fopenmp +# +# +################### +# +# Additional definitions that should not be changed +# +FLIBS = $(NETCDF_LIBRARY) +# BINDIR : directory for executables +BINDIR = $(ARCHDIR)/bin +# LIBBUILD : contains a directory for each library +LIBBUILD = $(ARCHDIR)/build/lib +# INCPSMILE : includes all *o and *mod for each library +INCPSMILE = -I$(LIBBUILD)/psmile.$(CHAN) -I$(LIBBUILD)/scrip -I$(LIBBUILD)/mct + +F90FLAGS = $(F90FLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2) +f90FLAGS = $(f90FLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2) +FFLAGS = $(FFLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2) +fFLAGS = $(fFLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2) +CCFLAGS = $(CCFLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2) +# +############################################################################# \ No newline at end of file diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index df4de0e..fbacc61 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -5,7 +5,34 @@ on: [push, pull_request] jobs: eclm_build_job: + name: ${{ matrix.config.name }} runs-on: ubuntu-22.04 + + strategy: + matrix: + config: + - { + name: "eCLM-Standalone", + use_oasis: "False", + coup_oas_icon: "False", + coup_oas_pfl: "False", + use_pdaf: "False" + } + - { + name: "eCLM-ParFlow-ICON", + use_oasis: "True", + coup_oas_icon: "True", + coup_oas_pfl: "True", + use_pdaf: "False" + } + - { + name: "eCLM-PDAF", + use_oasis: "False", + coup_oas_icon: "False", + coup_oas_pfl: "False", + use_pdaf: "True" + } + env: CC: mpicc FC: mpifort @@ -24,12 +51,41 @@ jobs: - name: Download MPI Fortran compiler run: sudo apt-get install gfortran openmpi-bin libopenmpi-dev + - if: matrix.config.use_oasis == 'True' + name: Cache OASIS3-MCT + uses: actions/cache@v4 + id: cache-deps + env: + cache-name: cache-eCLM-dependencies + with: + path: "~/.local" + key: cache-${{ matrix.config.name }} + + - if: matrix.config.use_oasis == 'True' && steps.cache-deps.outputs.cache-hit != 'true' + name: Install OASIS3-MCT + run: | + git clone https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git + cd oasis3-mct + export OASIS_ROOT=$(pwd) + echo "OASIS_ROOT=${OASIS_ROOT}" + echo "DEPENDENCIES_DIR=${DEPENDENCIES_DIR}" + cd util/make_dir + echo "include ${GITHUB_WORKSPACE}/.github/build.oasis3-mct.ubuntu22.04" > make.inc + cat make.inc + make realclean static-libs -f TopMakefileOasis3 + - name: Configure eCLM run: | - cmake -S src -B $BUILD_DIR \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ - -DCMAKE_C_COMPILER=$CC \ - -DCMAKE_Fortran_COMPILER=$FC + cmake -S src -B $BUILD_DIR \ + -DCMAKE_BUILD_TYPE="RELEASE" \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ + -DCMAKE_PREFIX_PATH="$HOME/.local" \ + -DCMAKE_C_COMPILER=$CC \ + -DCMAKE_Fortran_COMPILER=$FC \ + -DUSE_OASIS=${{ matrix.config.use_oasis }} \ + -DCOUP_OAS_ICON=${{ matrix.config.coup_oas_icon }} \ + -DCOUP_OAS_PFL=${{ matrix.config.coup_oas_pfl }} \ + -DUSE_PDAF=${{ matrix.config.use_pdaf }} - name: Build eCLM run: cmake --build $BUILD_DIR diff --git a/cmake/SetBuildOptions.cmake b/cmake/SetBuildOptions.cmake index 57b89c0..00f29b7 100644 --- a/cmake/SetBuildOptions.cmake +++ b/cmake/SetBuildOptions.cmake @@ -36,12 +36,30 @@ elseif(COMPILER STREQUAL "Intel" OR COMPILER STREQUAL "IntelLLVM") set(CMAKE_C_FLAGS_RELEASE "-O2 -debug minimal") set(CMAKE_Fortran_FLAGS "-free -qno-opt-dynamic-align -ftz -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model source -qopenmp") set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created") - set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal") + set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal") else() message(FATAL_ERROR "COMPILER='${COMPILER}' is not supported.") endif() +if(COUP_OAS_ICON) + list(APPEND COUPLED_MODELS "ICON") +endif() +if(COUP_OAS_PFL) + list(APPEND COUPLED_MODELS "ParFlow") +endif() +if(USE_PDAF) + list(APPEND COUPLED_MODELS "PDAF") +endif() + +if(COUPLED_MODELS) + list(PREPEND COUPLED_MODELS "eCLM") + list(JOIN COUPLED_MODELS "-" COUPLING_MODE) +else() + set(COUPLING_MODE "Standalone") +endif() + message(STATUS " ******* ${CMAKE_PROJECT_NAME} build options ******* ") message(STATUS " Build type = '${CMAKE_BUILD_TYPE}'") message(STATUS " Compiler = '${COMPILER}'") -message(STATUS " ********************************** ") +message(STATUS " Coupling mode = '${COUPLING_MODE}'") +message(STATUS " ********************************** ") \ No newline at end of file diff --git a/src/clm5/cpl/lnd_comp_mct.F90 b/src/clm5/cpl/lnd_comp_mct.F90 index a402546..56179be 100644 --- a/src/clm5/cpl/lnd_comp_mct.F90 +++ b/src/clm5/cpl/lnd_comp_mct.F90 @@ -405,6 +405,7 @@ subroutine lnd_run_mct(EClock, cdata_l, x2l_l, l2x_l) rof_prognostic=rof_prognostic, & glc_present=glc_present) +#ifndef COUP_OAS_ICON ! Map MCT to land data type ! Perform downscaling if appropriate @@ -418,6 +419,7 @@ subroutine lnd_run_mct(EClock, cdata_l, x2l_l, l2x_l) atm2lnd_inst = atm2lnd_inst, & glc2lnd_inst = glc2lnd_inst) call t_stopf ('lc_lnd_import') +#endif ! Use infodata to set orbital values if updated mid-run @@ -460,7 +462,15 @@ subroutine lnd_run_mct(EClock, cdata_l, x2l_l, l2x_l) if (nlend_sync .and. dosend) nlend = .true. #ifdef COUP_OAS_ICON - call oas_receive_icon(bounds, time_elapsed, atm2lnd_inst) + call oas_receive_icon(bounds, time_elapsed, x2l = x2l_l%rattr) + + call t_startf ('lc_lnd_import') + call lnd_import( bounds, & + x2l = x2l_l%rattr, & + glc_present = glc_present, & + atm2lnd_inst = atm2lnd_inst, & + glc2lnd_inst = glc2lnd_inst) + call t_stopf ('lc_lnd_import') #endif #ifdef COUP_OAS_PFL diff --git a/src/clm5/cpl/lnd_import_export.F90 b/src/clm5/cpl/lnd_import_export.F90 index 4923b72..b689924 100644 --- a/src/clm5/cpl/lnd_import_export.F90 +++ b/src/clm5/cpl/lnd_import_export.F90 @@ -204,6 +204,7 @@ subroutine lnd_import( bounds, x2l, glc_present, atm2lnd_inst, glc2lnd_inst) atm2lnd_inst%forc_rh_grc(g) = 100.0_r8*(forc_q / qsat) +#ifndef COUP_OAS_ICON ! Check that solar, specific-humidity and LW downward aren't negative if ( atm2lnd_inst%forc_lwrad_not_downscaled_grc(g) <= 0.0_r8 )then call endrun( sub//' ERROR: Longwave down sent from the atmosphere model is negative or zero' ) @@ -216,6 +217,7 @@ subroutine lnd_import( bounds, x2l, glc_present, atm2lnd_inst, glc2lnd_inst) if ( atm2lnd_inst%forc_q_not_downscaled_grc(g) < 0.0_r8 )then call endrun( sub//' ERROR: Bottom layer specific humidty sent from the atmosphere model is less than zero' ) end if +#endif ! Check if any input from the coupler is NaN if ( any(isnan(x2l(:,i))) )then diff --git a/src/clm5/oasis3/oas_defineMod.F90 b/src/clm5/oasis3/oas_defineMod.F90 index c58b61f..cdae0fc 100644 --- a/src/clm5/oasis3/oas_defineMod.F90 +++ b/src/clm5/oasis3/oas_defineMod.F90 @@ -21,6 +21,7 @@ subroutine oas_definitions_init(bounds) integer, allocatable :: partition(:) ! partition descriptor; input to oasis_def_partition integer :: gcell_start ! starting gridcell index integer :: gcell_previous ! gridcell index from previous loop iteration + integer :: num_local_gcells ! total number of gridcells for this process integer :: k, g ! array/loop indices integer :: grid_id ! id returned after call to oasis_def_partition #ifdef COUP_OAS_ICON @@ -39,38 +40,56 @@ subroutine oas_definitions_init(bounds) ! ----------------------------------------------------------------- ! ... Define partition ! ----------------------------------------------------------------- - ! partition length = (# elements for partition info) + (max segments ORANGE partition) x (# elements per segment info) - ! = 2 + 200*2 = 402 - allocate(partition(402)) - partition(:) = 0; k = 0 - - ! Use ORANGE partitioning scheme. This scheme defines an ensemble - ! of gridcell segments. See OASIS3-MCT User's guide for more info. - partition(1) = 3 - - ! Mark 1st segment - gcell_start = ldecomp%gdc2glo(bounds%begg) - partition(2) = 1 - gcell_previous = gcell_start - - ! Capture segments by detecting segment boundaries. A boundary is - ! detected when the current and previous gridcells are not consecutive. - do g = bounds%begg+1, bounds%endg - if (ldecomp%gdc2glo(g) - gcell_previous /= 1) then - ! Previous segment complete; its partition params could now be defined - partition(3+k) = gcell_start - 1 ! segment global offset (0-based) - partition(4+k) = gcell_previous - gcell_start + 1 ! segment length - k = k + 2 - gcell_start = ldecomp%gdc2glo(g) ! current gridcell marks the start of a new segment - partition(2) = partition(2) + 1 ! increment number of segments (limited to 200 based from OASIS3-MCT User's guide) - end if - gcell_previous = ldecomp%gdc2glo(g) - enddo - - ! Define partition params for last segment - partition(3+k) = gcell_start - 1 - partition(4+k) = gcell_previous - gcell_start + 1 - partition(2) = partition(2) + 1 + if (ldomain%nj == 1) then + ! POINTS partitioning scheme + ! partition length = 2 + total number of gridpoints allocated to this MPI process + num_local_gcells = bounds%endg - bounds%begg + 1 + allocate(partition(num_local_gcells + 2)) + partition(:) = 0; k = 0 + + ! Use POINTS partitioning scheme. This partition is a list of global indices associated with each process. + partition(1) = 4 + partition(2) = num_local_gcells + + do g = bounds%begg, bounds%endg + partition(3+k) = ldecomp%gdc2glo(g) + k = k + 1 + enddo + else + ! ORANGE partitioning scheme + ! partition length = (# elements for partition info) + (max segments ORANGE partition) x (# elements per segment info) + ! = 2 + 200*2 = 402 + allocate(partition(402)) + partition(:) = 0; k = 0 + + ! Use ORANGE partitioning scheme. This scheme defines an ensemble + ! of gridcell segments. See OASIS3-MCT User's guide for more info. + partition(1) = 3 + + ! Mark 1st segment + gcell_start = ldecomp%gdc2glo(bounds%begg) + partition(2) = 1 + gcell_previous = gcell_start + + ! Capture segments by detecting segment boundaries. A boundary is + ! detected when the current and previous gridcells are not consecutive. + do g = bounds%begg+1, bounds%endg + if (ldecomp%gdc2glo(g) - gcell_previous /= 1) then + ! Previous segment complete; its partition params could now be defined + partition(3+k) = gcell_start - 1 ! segment global offset (0-based) + partition(4+k) = gcell_previous - gcell_start + 1 ! segment length + k = k + 2 + gcell_start = ldecomp%gdc2glo(g) ! current gridcell marks the start of a new segment + partition(2) = partition(2) + 1 ! increment number of segments (limited to 200 based from OASIS3-MCT User's guide) + end if + gcell_previous = ldecomp%gdc2glo(g) + enddo + + ! Define partition params for last segment + partition(3+k) = gcell_start - 1 + partition(4+k) = gcell_previous - gcell_start + 1 + partition(2) = partition(2) + 1 + end if call oasis_def_partition(grid_id, partition, ierror, ldomain%ns) deallocate(partition) diff --git a/src/clm5/oasis3/oas_sendReceiveMod.F90 b/src/clm5/oasis3/oas_sendReceiveMod.F90 index 4497223..abd8db7 100644 --- a/src/clm5/oasis3/oas_sendReceiveMod.F90 +++ b/src/clm5/oasis3/oas_sendReceiveMod.F90 @@ -53,43 +53,43 @@ end subroutine oas_send #endif #ifdef COUP_OAS_ICON - subroutine oas_receive_icon(bounds, seconds_elapsed, atm2lnd_inst) + subroutine oas_receive_icon(bounds, seconds_elapsed, x2l) use atm2lndType, only: atm2lnd_type + use clm_cpl_indices type(bounds_type), intent(in) :: bounds integer , intent(in) :: seconds_elapsed - type(atm2lnd_type), intent(inout) :: atm2lnd_inst + real(r8) , intent(inout) :: x2l(:,:) ! driver import state to land model real(kind=r8), allocatable :: buffer(:,:) integer :: num_grid_points integer :: info - integer :: g + integer :: g, i num_grid_points = (bounds%endg - bounds%begg) + 1 allocate(buffer(num_grid_points, 1)) ! call oasis_get(oas_id_t, seconds_elapsed, oas_rcv_meta(:,:,oas_id_t), info) - call oasis_get(oas_id_t, seconds_elapsed, atm2lnd_inst%forc_t_not_downscaled_grc, info) - call oasis_get(oas_id_u, seconds_elapsed, atm2lnd_inst%forc_u_grc, info) - call oasis_get(oas_id_v, seconds_elapsed, atm2lnd_inst%forc_v_grc, info) - call oasis_get(oas_id_qv, seconds_elapsed, atm2lnd_inst%forc_q_not_downscaled_grc, info) - call oasis_get(oas_id_ht, seconds_elapsed, atm2lnd_inst%forc_hgt_grc, info) - call oasis_get(oas_id_pr, seconds_elapsed, atm2lnd_inst%forc_pbot_not_downscaled_grc, info) - call oasis_get(oas_id_rs, seconds_elapsed, atm2lnd_inst%forc_solad_grc(:,1), info) - call oasis_get(oas_id_fs, seconds_elapsed, atm2lnd_inst%forc_solai_grc(:,1), info) - call oasis_get(oas_id_lw, seconds_elapsed, atm2lnd_inst%forc_lwrad_not_downscaled_grc, info) - call oasis_get(oas_id_cr, seconds_elapsed, atm2lnd_inst%forc_rain_not_downscaled_grc, info) - call oasis_get(oas_id_gr, seconds_elapsed, atm2lnd_inst%forc_snow_not_downscaled_grc, info) - - !SPo: some postprocessing of atm2lnd is missing; may better use x2l + call oasis_get(oas_id_t, seconds_elapsed, x2l(index_x2l_Sa_tbot,:), info) + call oasis_get(oas_id_u, seconds_elapsed, x2l(index_x2l_Sa_u,:), info) + call oasis_get(oas_id_v, seconds_elapsed, x2l(index_x2l_Sa_v,:), info) + call oasis_get(oas_id_qv, seconds_elapsed, x2l(index_x2l_Sa_shum,:), info) + call oasis_get(oas_id_ht, seconds_elapsed, x2l(index_x2l_Sa_z,:), info) + call oasis_get(oas_id_pr, seconds_elapsed, x2l(index_x2l_Sa_pbot,:), info) + call oasis_get(oas_id_rs, seconds_elapsed, x2l(index_x2l_Faxa_swvdr,:), info) + call oasis_get(oas_id_fs, seconds_elapsed, x2l(index_x2l_Faxa_swvdf,:), info) + call oasis_get(oas_id_lw, seconds_elapsed, x2l(index_x2l_Faxa_lwdn,:), info) + call oasis_get(oas_id_cr, seconds_elapsed, x2l(index_x2l_Faxa_rainl,:), info) + call oasis_get(oas_id_gr, seconds_elapsed, x2l(index_x2l_Faxa_snowl,:), info) + x2l(index_x2l_Faxa_rainc,:) = 0. + x2l(index_x2l_Faxa_snowc,:) = 0. do g=bounds%begg,bounds%endg - atm2lnd_inst%forc_solad_grc(g,1) = 0.5_r8 * atm2lnd_inst%forc_solad_grc(g,1) - atm2lnd_inst%forc_solad_grc(g,2) = atm2lnd_inst%forc_solad_grc(g,1) - atm2lnd_inst%forc_solai_grc(g,1) = 0.5_r8 * atm2lnd_inst%forc_solai_grc(g,1) - atm2lnd_inst%forc_solai_grc(g,2) = atm2lnd_inst%forc_solai_grc(g,1) - atm2lnd_inst%forc_solar_grc(g) = atm2lnd_inst%forc_solad_grc(g,2) + atm2lnd_inst%forc_solad_grc(g,1) & - + atm2lnd_inst%forc_solai_grc(g,2) + atm2lnd_inst%forc_solai_grc(g,1) + i = 1 + (g - bounds%begg) + x2l(index_x2l_Faxa_swvdr,i) = 0.5_r8 * x2l(index_x2l_Faxa_swvdr,i) + x2l(index_x2l_Faxa_swndr,i) = x2l(index_x2l_Faxa_swvdr,i) + x2l(index_x2l_Faxa_swvdf,i) = 0.5_r8 * x2l(index_x2l_Faxa_swvdf,i) + x2l(index_x2l_Faxa_swndf,i) = x2l(index_x2l_Faxa_swvdf,i) enddo end subroutine oas_receive_icon diff --git a/src/csm_share/mct/seq_comm_mct.F90 b/src/csm_share/mct/seq_comm_mct.F90 index 90dfd6a..5da0027 100644 --- a/src/csm_share/mct/seq_comm_mct.F90 +++ b/src/csm_share/mct/seq_comm_mct.F90 @@ -611,15 +611,11 @@ subroutine comp_comm_init(driver_comm, comp_rootpe, comp_nthreads, comp_layout, pelist(2,1) = cmax(n) pelist(3,1) = cstr(n) endif -#ifdef USE_PDAF - !write(*,*) "before seq_comm_set_comm bcast", pelist, DRIVER_COMM -#endif call mpi_bcast(pelist, size(pelist), MPI_INTEGER, 0, DRIVER_COMM, ierr) + #ifdef USE_PDAF if (present(pdaf_id) .and. present(pdaf_max)) then - !write(*,*) "before seq_comm_setcomm", pdaf_id, pdaf_max, mype, n, & - ! COMPID(n), pelist, comp_nthreads, name - call seq_comm_setcomm(COMPID(n),pelist,nthreads=comp_nthreads,iname=name,inst=pdaf_id,tinst=pdaf_max) + call seq_comm_setcomm(COMPID(n),pelist,nthreads=comp_nthreads,iname=name,inst=pdaf_id,tinst=pdaf_max) else if (present(drv_comm_id)) then #else if (present(drv_comm_id)) then diff --git a/src/eclm/cime_comp_mod.F90 b/src/eclm/cime_comp_mod.F90 index a1fefdd..6215eed 100644 --- a/src/eclm/cime_comp_mod.F90 +++ b/src/eclm/cime_comp_mod.F90 @@ -43,6 +43,7 @@ module cime_comp_mod #if defined(USE_OASIS) use mod_oasis #endif + !---------------------------------------------------------------------------- ! component model interfaces (init, run, final methods) !---------------------------------------------------------------------------- @@ -179,9 +180,7 @@ module cime_comp_mod implicit none -#ifndef USE_PDAF private -#endif public cime_pre_init1, cime_pre_init2, cime_init, cime_run, cime_final public timing_dir, mpicom_GLOID @@ -591,24 +590,26 @@ subroutine cime_pre_init1(esmf_log_option) #endif use shr_pio_mod, only : shr_pio_init1, shr_pio_init2 use seq_comm_mct, only: num_inst_driver - !---------------------------------------------------------- !| Initialize MCT and MPI communicators and IO !---------------------------------------------------------- character(CS), intent(out) :: esmf_log_option ! For esmf_logfile_kind - #ifdef USE_PDAF integer, optional, intent(in) :: pdaf_comm integer, optional, intent(in) :: pdaf_id integer, optional, intent(in) :: pdaf_max #endif + integer, dimension(num_inst_total) :: comp_id, comp_comm, comp_comm_iam logical :: comp_iamin(num_inst_total) character(len=seq_comm_namelen) :: comp_name(num_inst_total) integer :: it - integer :: driver_id, oas_comp_id + integer :: driver_id integer :: driver_comm +#if defined(USE_OASIS) + integer :: oas_comp_id +#endif #ifndef USE_PDAF call mpi_init(ierr) @@ -628,7 +629,6 @@ subroutine cime_pre_init1(esmf_log_option) #ifdef USE_PDAF if (present(pdaf_comm)) then global_comm = pdaf_comm - !write(*,*) "PDAF_COMM present" else call mpi_comm_dup(MPI_COMM_WORLD, global_comm, ierr) call shr_mpi_chkerr(ierr,subname//' mpi_comm_dup') @@ -2186,12 +2186,21 @@ end subroutine cime_init !******************************************************************************* !=============================================================================== +#ifdef USE_PDAF + subroutine cime_run(ntsteps) +#else subroutine cime_run() +#endif use seq_comm_mct, only: atm_layout, lnd_layout, ice_layout, glc_layout, & rof_layout, ocn_layout, wav_layout, esp_layout use shr_string_mod, only: shr_string_listGetIndexF use seq_comm_mct, only: num_inst_driver +#ifdef USE_PDAF + ! TSMP specific + integer, intent(in), optional :: ntsteps + integer :: counter=0 +#endif ! gptl timer lookup variables integer, parameter :: hashcnt=7 integer :: hashint(hashcnt) @@ -2243,6 +2252,19 @@ subroutine cime_run() call t_stopf ('CPL:RUN_LOOP_BSTART') Time_begin = mpi_wtime() Time_bstep = mpi_wtime() + +#ifdef USE_PDAF + ! Check for optional input `ntsteps` + if(.not. present(ntsteps)) then + write(logunit,*) 'ERROR: ntsteps input not present, but needed for TSMP-PDAF ;' + call shr_sys_abort(subname// & + ' missing ntsteps input that is needed for TSMP-PDAF') + end if + + ! Explicitly set `counter` to zero before loop + counter = 0 +#endif + do while ( .not. stop_alarm) call t_startf('CPL:RUN_LOOP', hashint(1)) @@ -4086,6 +4108,18 @@ subroutine cime_run() call t_drvstopf ('CPL:BARRIERALARM',cplrun=.true.) endif +#ifdef USE_PDAF + ! TSMP specific stop condition: + counter = counter + 1 + if (present(ntsteps) .and. counter == ntsteps) then + if (iamroot_CPLID) then + write(logunit,*) ' ' + write(logunit,103) subname,' NOTE: Stopping from TSMP-PDAF alarm ntsteps' + write(logunit,*) ' ' + endif + stop_alarm = .true. + end if +#endif enddo ! driver run loop !|---------------------------------------------------------- @@ -4320,10 +4354,9 @@ subroutine cime_cpl_init(comm_in, comm_out, num_inst_driver, id) end if #ifdef USE_PDAF - !write(*,*) "just before split", comm_in, pdaf_id, mype, numpes, comm_out if (pdaf_max > 1) then - call mpi_comm_split(comm_in, pdaf_id, 0, comm_out, ierr) - call shr_mpi_chkerr(ierr,subname//' mpi_comm_split') + call mpi_comm_split(comm_in, pdaf_id, 0, comm_out, ierr) + call shr_mpi_chkerr(ierr,subname//' mpi_comm_split') else if (num_inst_driver == 1) then #else if (num_inst_driver == 1) then