From 6e393f6edc6e3097c5e0be60848eb4b8ddfe1d73 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 15 Jun 2024 23:06:57 -0500 Subject: [PATCH 1/4] data ice branch from data rof case --res T62_oQU120 --compset CMPASO-NYF data ice is similar so far with data atm, ocean and rof an ice_domain file is initialized and used by moab driver --- .../data_comps/dice/src/dice_comp_mod.F90 | 181 +++++++++++++++++- .../data_comps/dice/src/ice_comp_mct.F90 | 36 +++- driver-moab/main/cplcomp_exchange_mod.F90 | 82 +++++--- driver-moab/shr/seq_infodata_mod.F90 | 15 +- 4 files changed, 279 insertions(+), 35 deletions(-) diff --git a/components/data_comps/dice/src/dice_comp_mod.F90 b/components/data_comps/dice/src/dice_comp_mod.F90 index 8f3afd70bf87..94dd0c012433 100644 --- a/components/data_comps/dice/src/dice_comp_mod.F90 +++ b/components/data_comps/dice/src/dice_comp_mod.F90 @@ -33,7 +33,10 @@ module dice_comp_mod use dice_shr_mod , only: flux_Qacc0 ! namelist input -initial water accumulation value use dice_shr_mod , only: nullstr use dice_flux_atmice_mod, only: dice_flux_atmice - +#ifdef HAVE_MOAB + use seq_comm_mct, only : MPSIID ! id of moab ice app + use iso_c_binding +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -111,6 +114,12 @@ subroutine dice_comp_init(Eclock, x2i, i2x, & inst_suffix, inst_name, logunit, read_restart, & scmMode, scmlat, scmlon) +#ifdef HAVE_MOAB + use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, & + iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & + iMOAB_GetMeshInfo, iMOAB_UpdateMeshInfo, iMOAB_WriteMesh +#endif ! !DESCRIPTION: initialize dice model implicit none @@ -141,7 +150,17 @@ subroutine dice_comp_init(Eclock, x2i, i2x, & logical :: exists ! file existance logical integer(IN) :: nu ! unit number character(CL) :: calendar ! calendar type - +#ifdef HAVE_MOAB + character*400 tagname + real(R8) latv, lonv + integer iv, tagindex, ilat, ilon, ierr !, arrsize, nfields + real(R8), allocatable, target :: data(:) + integer(IN), pointer :: idata(:) ! temporary + real(r8), dimension(:), allocatable :: moab_vert_coords ! temporary +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif +#endif !--- formats --- character(*), parameter :: F00 = "('(dice_comp_init) ',8a)" character(*), parameter :: F0L = "('(dice_comp_init) ',a, l2)" @@ -219,6 +238,120 @@ subroutine dice_comp_init(Eclock, x2i, i2x, & call shr_dmodel_rearrGGrid(SDICE%grid, ggrid, gsmap, rearr, mpicom) call t_stopf('dice_initmctdom') +#ifdef HAVE_MOAB + ilat = mct_aVect_indexRA(ggrid%data,'lat') + ilon = mct_aVect_indexRA(ggrid%data,'lon') + allocate(moab_vert_coords(lsize*3)) + do iv = 1, lsize + lonv = ggrid%data%rAttr(ilon, iv) * SHR_CONST_PI/180. + latv = ggrid%data%rAttr(ilat, iv) * SHR_CONST_PI/180. + moab_vert_coords(3*iv-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*iv-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*iv )=SIN(latv) + enddo + + ! create the vertices with coordinates from MCT domain + ierr = iMOAB_CreateVertices(MPSIID, lsize*3, 3, moab_vert_coords) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to create MOAB vertices in data ice model') + + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(MPSIID, tagname, & + 0, & ! dense, integer + 1, & ! number of components + tagindex ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to retrieve GLOBAL_ID tag ') + + ! get list of global IDs for Dofs + call mct_gsMap_orderedPoints(gsMap, my_task, idata) + + ierr = iMOAB_SetIntTagStorage ( MPSIID, tagname, lsize, & + 0, & ! vertex type + idata) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to set GLOBAL_ID tag ') + + ierr = iMOAB_ResolveSharedEntities( MPSIID, lsize, idata ); + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to resolve shared entities') + + deallocate(moab_vert_coords) + deallocate(idata) + + ierr = iMOAB_UpdateMeshInfo( MPSIID ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to update mesh info ') + + allocate(data(lsize)) + ierr = iMOAB_DefineTagStorage( MPSIID, "area:aream:frac:mask"//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create tag: area:aream:frac:mask' ) + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'area'),:) + tagname='area'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( MPSIID, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to get area tag ') + + ! set the same data for aream (model area) as area + ! data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'aream'),:) + tagname='aream'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( MPSIID, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set aream tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'mask'),:) + tagname='mask'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( MPSIID, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set mask tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'frac'),:) + tagname='frac'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( MPSIID, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set frac tag ') + + deallocate(data) + + ! define tags + ierr = iMOAB_DefineTagStorage( MPSIID, trim(seq_flds_x2i_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_x2i_fields tags ') + + ierr = iMOAB_DefineTagStorage( MPSIID, trim(seq_flds_i2x_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_i2x_fields tags ') +#ifdef MOABDEBUG + ! debug test + outfile = 'IceDataMesh.h5m'//C_NULL_CHAR + wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ! + ! write out the mesh file to disk + ierr = iMOAB_WriteMesh(MPSIID, trim(outfile), trim(wopts)) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' ERROR in writing data mesh ice ') + endif +#endif +#endif + !---------------------------------------------------------------------------- ! Initialize MCT attribute vectors !---------------------------------------------------------------------------- @@ -388,6 +521,13 @@ subroutine dice_comp_run(EClock, x2i, i2x, & inst_suffix, logunit, read_restart, case_name) use shr_cal_mod, only : shr_cal_ymdtod2string ! !DESCRIPTION: run method for dice model +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif +#ifdef HAVE_MOAB + use seq_flds_mod , only: seq_flds_i2x_fields + use seq_flds_mod , only: moab_set_tag_from_av +#endif implicit none ! !INPUT/OUTPUT PARAMETERS: @@ -423,6 +563,17 @@ subroutine dice_comp_run(EClock, x2i, i2x, & logical :: write_restart ! restart now character(len=18) :: date_str +#ifdef HAVE_MOAB + real(R8), allocatable, target :: datam(:) + type(mct_list) :: temp_list + integer :: size_list, index_list + type(mct_string) :: mctOStr ! + character*400 tagname, mct_field +#ifdef MOABDEBUG + integer :: cur_dice_stepno, ierr + character*100 outfile, wopts, lnum +#endif +#endif character(*), parameter :: F00 = "('(dice_comp_run) ',8a)" character(*), parameter :: F04 = "('(dice_comp_run) ',2a,2i8,'s')" character(*), parameter :: subName = "(dice_comp_run) " @@ -671,6 +822,32 @@ subroutine dice_comp_run(EClock, x2i, i2x, & call t_stopf('dice') +#ifdef HAVE_MOAB + lsize = mct_avect_lsize(i2x) ! is it the same as mct_avect_lsize(avstrm) ? + allocate(datam(lsize)) ! + call mct_list_init(temp_list ,seq_flds_i2x_fields) + size_list=mct_list_nitem (temp_list) + do index_list = 1, size_list + call mct_list_get(mctOStr,index_list,temp_list) + mct_field = mct_string_toChar(mctOStr) + tagname= trim(mct_field)//C_NULL_CHAR + call moab_set_tag_from_av(tagname, i2x, index_list, MPSIID, datam, lsize) ! loop over all a2x fields, not just a few + enddo + call mct_list_clean(temp_list) + deallocate(datam) ! maybe we should keep it around, deallocate at the final only? + +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_dice_stepno ) + write(lnum,"(I0.2)")cur_dice_stepno + outfile = 'dice_comp_run_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(MPSIID, outfile, wopts) + if (ierr > 0 ) then + write(logunit,*) 'Failed to write data ice component state ' + endif +#endif + +#endif !---------------------------------------------------------------------------- ! Log output for model date !---------------------------------------------------------------------------- diff --git a/components/data_comps/dice/src/ice_comp_mct.F90 b/components/data_comps/dice/src/ice_comp_mct.F90 index e409e5989317..b38445ad2dc4 100644 --- a/components/data_comps/dice/src/ice_comp_mct.F90 +++ b/components/data_comps/dice/src/ice_comp_mct.F90 @@ -16,7 +16,11 @@ module ice_comp_mct use dice_comp_mod , only: dice_comp_init, dice_comp_run, dice_comp_final use dice_shr_mod , only: dice_shr_read_namelists use seq_flds_mod , only: seq_flds_i2x_fields, seq_flds_x2i_fields, seq_flds_i2o_per_cat - +#ifdef HAVE_MOAB + use seq_comm_mct, only : MPSIID ! iMOAB app id for ice + use iso_c_binding + use iMOAB , only: iMOAB_RegisterApplication +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -51,7 +55,9 @@ module ice_comp_mct !=============================================================================== subroutine ice_init_mct( EClock, cdata, x2i, i2x, NLFilename ) - +#ifdef HAVE_MOAB + use shr_stream_mod, only: shr_stream_getDomainInfo, shr_stream_getFile +#endif ! !DESCRIPTION: initialize dice model implicit none @@ -73,6 +79,16 @@ subroutine ice_init_mct( EClock, cdata, x2i, i2x, NLFilename ) logical :: scmMode = .false. ! single column mode real(R8) :: scmLat = shr_const_SPVAL ! single column lat real(R8) :: scmLon = shr_const_SPVAL ! single column lon +#ifdef HAVE_MOAB + character(CL) :: filePath ! generic file path + character(CL) :: fileName ! generic file name + character(CS) :: timeName ! domain file: time variable name + character(CS) :: lonName ! domain file: lon variable name + character(CS) :: latName ! domain file: lat variable name + character(CS) :: hgtName ! domain file: hgt variable name + character(CS) :: maskName ! domain file: mask variable name + character(CS) :: areaName ! domain file: area variable name +#endif character(*), parameter :: subName = "(ice_init_mct) " !------------------------------------------------------------------------------- @@ -146,6 +162,22 @@ subroutine ice_init_mct( EClock, cdata, x2i, i2x, NLFilename ) inst_suffix, inst_name, logunit, read_restart, & scmMode, scmlat, scmlon) + +#ifdef HAVE_MOAB + ierr = iMOAB_RegisterApplication(trim("DICE")//C_NULL_CHAR, mpicom, compid, MPSIID) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in registering data ice comp' + call shr_sys_abort(subname//' ERROR in registering data ice comp') + endif + if (my_task == master_task) then + call shr_stream_getDomainInfo(SDICE%stream(1), filePath,fileName,timeName,lonName, & + latName,hgtName,maskName,areaName) + call shr_stream_getFile(filePath,fileName) + ! send path of ice domain to MOAB coupler. + call seq_infodata_PutData( infodata, ice_domain=fileName) + write(logunit,*), ' filename: ', filename + endif +#endif !---------------------------------------------------------------------------- ! Fill infodata that needs to be returned from dice !---------------------------------------------------------------------------- diff --git a/driver-moab/main/cplcomp_exchange_mod.F90 b/driver-moab/main/cplcomp_exchange_mod.F90 index 0e2682192697..e21d3f4554a2 100644 --- a/driver-moab/main/cplcomp_exchange_mod.F90 +++ b/driver-moab/main/cplcomp_exchange_mod.F90 @@ -1020,6 +1020,7 @@ subroutine cplcomp_moab_Init(infodata,comp) character(CL) :: rtm_mesh, rof_domain character(CL) :: lnd_domain character(CL) :: ocn_domain + character(CL) :: ice_domain ! used for data ice only? character(CL) :: atm_mesh integer :: maxMH, maxMPO, maxMLID, maxMSID, maxMRID ! max pids for moab apps atm, ocn, lnd, sea-ice, rof integer :: tagtype, numco, tagindex, partMethod, nghlay @@ -1543,37 +1544,60 @@ subroutine cplcomp_moab_Init(infodata,comp) if (comp%oneletterid == 'i' .and. maxMSID /= -1) then call seq_comm_getinfo(cplid ,mpigrp=mpigrp_cplid) ! receiver group call seq_comm_getinfo(id_old,mpigrp=mpigrp_old) ! component group pes - + ! find ice domain file if it exists; it would be for data ice model (ice_prognostic false) + call seq_infodata_GetData(infodata,ice_domain=ice_domain) if (MPI_COMM_NULL /= mpicom_old ) then ! it means we are on the component p #ifdef MOABDEBUG - outfile = 'wholeSeaIce.h5m'//C_NULL_CHAR - wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR - ierr = iMOAB_WriteMesh(MPSIID, outfile, wopts) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in writing sea-ice' - call shr_sys_abort(subname//' ERROR in writing sea-ice') - endif + outfile = 'wholeSeaIce.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(MPSIID, outfile, wopts) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in writing sea-ice' + call shr_sys_abort(subname//' ERROR in writing sea-ice') + endif #endif ! start copy from ocean code - ! send sea ice mesh to coupler - ierr = iMOAB_SendMesh(MPSIID, mpicom_join, mpigrp_cplid, id_join, partMethod) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in sending sea ice mesh to coupler ' - call shr_sys_abort(subname//' ERROR in sending sea ice mesh to coupler ') - endif - if (MPSIID >= 0) then - ierr = iMOAB_GetMeshInfo ( MPSIID, nvert, nvise, nbl, nsurf, nvisBC ) - comp%mbApCCid = MPSIID ! phys atm - comp%mbGridType = 1 ! 0 or 1, pc or cells - comp%mblsize = nvise(1) ! vertices - endif - - endif + if (MPSIID >= 0) then + ierr = iMOAB_GetMeshInfo ( MPSIID, nvert, nvise, nbl, nsurf, nvisBC ) + comp%mbApCCid = MPSIID ! ice imoab app id + endif + if ( trim(ice_domain) == 'none' ) then ! regular ice model + comp%mbGridType = 1 ! 0 or 1, pc or cells + comp%mblsize = nvise(1) ! cells + ! send sea ice mesh to coupler + ierr = iMOAB_SendMesh(MPSIID, mpicom_join, mpigrp_cplid, id_join, partMethod) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in sending sea ice mesh to coupler ' + call shr_sys_abort(subname//' ERROR in sending sea ice mesh to coupler ') + endif + else + comp%mbGridType = 0 ! 0 or 1, pc or cells + comp%mblsize = nvert(1) ! vertices + endif + endif if (MPI_COMM_NULL /= mpicom_new ) then ! we are on the coupler pes appname = "COUPLE_MPASSI"//C_NULL_CHAR ! migrated mesh gets another app id, moab moab sea ice to coupler (mbix) ierr = iMOAB_RegisterApplication(trim(appname), mpicom_new, id_join, mbixid) - ierr = iMOAB_ReceiveMesh(mbixid, mpicom_join, mpigrp_old, id_old) + if ( trim(ice_domain) == 'none' ) then ! regular ice model + ierr = iMOAB_ReceiveMesh(mbixid, mpicom_join, mpigrp_old, id_old) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in receiving ice mesh in coupler ' + call shr_sys_abort(subname//' ERROR in receiving sea ice mesh in coupler ') + endif + else + ! we need to read the mesh ice (domain file) + ierr = iMOAB_LoadMesh(mbixid, trim(ice_domain)//C_NULL_CHAR, & + "PARALLEL=READ_PART;PARTITION_METHOD=SQIJ;VARIABLE=;NO_CULLING;REPARTITION", 0) + if ( ierr /= 0 ) then + write(logunit,*) 'Failed to load ice domain mesh on coupler' + call shr_sys_abort(subname//' ERROR Failed to load ice domain mesh on coupler ') + endif + if (seq_comm_iamroot(CPLID)) then + write(logunit,'(A)') subname//' load ice domain mesh from file '//trim(ice_domain) + endif + + endif tagtype = 1 ! dense, double numco = 1 ! one value per cell / entity tagname = trim(seq_flds_i2x_fields)//C_NULL_CHAR @@ -1626,11 +1650,13 @@ subroutine cplcomp_moab_Init(infodata,comp) #endif endif if (MPSIID .ge. 0) then ! we are on component sea ice pes - context_id = id_join - ierr = iMOAB_FreeSenderBuffers(MPSIID, context_id) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in freeing buffers ' - call shr_sys_abort(subname//' ERROR in freeing buffers ') + if ( trim(ice_domain) == 'none' ) then + context_id = id_join + ierr = iMOAB_FreeSenderBuffers(MPSIID, context_id) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in freeing buffers ' + call shr_sys_abort(subname//' ERROR in freeing buffers ') + endif endif endif diff --git a/driver-moab/shr/seq_infodata_mod.F90 b/driver-moab/shr/seq_infodata_mod.F90 index 2e9838bc69c5..512adb853b63 100644 --- a/driver-moab/shr/seq_infodata_mod.F90 +++ b/driver-moab/shr/seq_infodata_mod.F90 @@ -234,6 +234,7 @@ MODULE seq_infodata_mod character(SHR_KIND_CL) :: rof_mesh ! path to river mesh file character(SHR_KIND_CL) :: rof_domain ! path to river domain file; only for data rof for now character(SHR_KIND_CL) :: ocn_domain ! path to ocean domain file, used by data ocean models only + character(SHR_KIND_CL) :: ice_domain ! path to ice domain file, used by data ice models only character(SHR_KIND_CL) :: atm_mesh ! path to atmosphere domain/mesh file, used by data atm models only !--- set via components and may be time varying --- @@ -795,6 +796,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%rof_mesh = 'none' infodata%rof_domain = 'none' infodata%ocn_domain = 'none' ! will be used for ocean data models only; will be used as a signal + infodata%ice_domain = 'none' ! will be used for ice data models only; will be used as a signal infodata%atm_mesh = 'none' ! will be used for atmosphere data models only; will be used as a signal ! not sure if it exists always actually @@ -1040,7 +1042,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ wav_phase, iac_phase, esp_phase, wav_nx, wav_ny, atm_nx, atm_ny, & lnd_nx, lnd_ny, rof_nx, rof_ny, ice_nx, ice_ny, ocn_nx, ocn_ny, & iac_nx, iac_ny, glc_nx, glc_ny, lnd_domain, rof_mesh, rof_domain, & - ocn_domain, atm_mesh, eps_frac, & + ocn_domain, ice_domain, atm_mesh, eps_frac, & eps_amask, eps_agrid, eps_aarea, eps_omask, eps_ogrid, eps_oarea, & reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & @@ -1216,6 +1218,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ character(SHR_KIND_CL), optional, intent(OUT) :: rof_mesh character(SHR_KIND_CL), optional, intent(OUT) :: rof_domain character(SHR_KIND_CL), optional, intent(OUT) :: ocn_domain + character(SHR_KIND_CL), optional, intent(OUT) :: ice_domain character(SHR_KIND_CL), optional, intent(OUT) :: atm_mesh real(SHR_KIND_R8), optional, intent(OUT) :: nextsw_cday ! calendar of next atm shortwave @@ -1406,6 +1409,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(rof_mesh) ) rof_mesh = infodata%rof_mesh if ( present(rof_domain) ) rof_domain = infodata%rof_domain if ( present(ocn_domain) ) ocn_domain = infodata%ocn_domain + if ( present(ice_domain) ) ice_domain = infodata%ice_domain if ( present(atm_mesh) ) atm_mesh = infodata%atm_mesh if ( present(nextsw_cday) ) nextsw_cday = infodata%nextsw_cday @@ -1602,8 +1606,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ wav_phase, iac_phase, esp_phase, wav_nx, wav_ny, atm_nx, atm_ny, & lnd_nx, lnd_ny, rof_nx, rof_ny, ice_nx, ice_ny, ocn_nx, ocn_ny, & iac_nx, iac_ny, glc_nx, glc_ny, eps_frac, eps_amask, lnd_domain, & - rof_mesh, rof_domain, ocn_domain, atm_mesh, eps_agrid, eps_aarea, & - eps_omask, eps_ogrid, eps_oarea, & + rof_mesh, rof_domain, ocn_domain, ice_domain, atm_mesh, eps_agrid, & + eps_aarea, eps_omask, eps_ogrid, eps_oarea, & reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & mct_usealltoall, mct_usevector, glc_valid_input, nlmaps_verbosity) @@ -1777,6 +1781,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ character(SHR_KIND_CL), optional, intent(IN) :: rof_mesh character(SHR_KIND_CL), optional, intent(IN) :: rof_domain character(SHR_KIND_CL), optional, intent(IN) :: ocn_domain + character(SHR_KIND_CL), optional, intent(IN) :: ice_domain character(SHR_KIND_CL), optional, intent(IN) :: atm_mesh real(SHR_KIND_R8), optional, intent(IN) :: nextsw_cday ! calendar of next atm shortwave @@ -1966,6 +1971,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(rof_mesh) ) infodata%rof_mesh = rof_mesh if ( present(rof_domain) ) infodata%rof_domain = rof_domain if ( present(ocn_domain) ) infodata%ocn_domain = ocn_domain + if ( present(ice_domain) ) infodata%ice_domain = ice_domain if ( present(atm_mesh) ) infodata%atm_mesh = atm_mesh if ( present(nextsw_cday) ) infodata%nextsw_cday = nextsw_cday @@ -2279,6 +2285,7 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%rof_mesh, mpicom) call shr_mpi_bcast(infodata%rof_domain, mpicom) call shr_mpi_bcast(infodata%ocn_domain, mpicom) + call shr_mpi_bcast(infodata%ice_domain, mpicom) call shr_mpi_bcast(infodata%atm_mesh, mpicom) call shr_mpi_bcast(infodata%nextsw_cday, mpicom) call shr_mpi_bcast(infodata%precip_fact, mpicom) @@ -2552,6 +2559,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ice_nx, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ice_ny, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ice_domain, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) @@ -3000,6 +3008,7 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0I) subname,'rof_mesh = ', infodata%rof_mesh write(logunit,F0I) subname,'rof_domain = ', infodata%rof_domain write(logunit,F0I) subname,'ocn_domain = ', infodata%ocn_domain + write(logunit,F0I) subname,'ice_domain = ', infodata%ice_domain write(logunit,F0I) subname,'atm_mesh = ', infodata%atm_mesh write(logunit,F0R) subname,'nextsw_cday = ', infodata%nextsw_cday From ec58e8216ad976371235f6844bcf065eac452baf Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sun, 16 Jun 2024 10:31:15 -0500 Subject: [PATCH 2/4] define data ice imoab app before init --- components/data_comps/dice/src/ice_comp_mct.F90 | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/components/data_comps/dice/src/ice_comp_mct.F90 b/components/data_comps/dice/src/ice_comp_mct.F90 index b38445ad2dc4..f8c48c89240a 100644 --- a/components/data_comps/dice/src/ice_comp_mct.F90 +++ b/components/data_comps/dice/src/ice_comp_mct.F90 @@ -155,7 +155,13 @@ subroutine ice_init_mct( EClock, cdata, x2i, i2x, NLFilename ) !---------------------------------------------------------------------------- ! Initialize dice !---------------------------------------------------------------------------- - +#ifdef HAVE_MOAB + ierr = iMOAB_RegisterApplication(trim("DICE")//C_NULL_CHAR, mpicom, compid, MPSIID) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in registering data ice comp' + call shr_sys_abort(subname//' ERROR in registering data ice comp') + endif +#endif call dice_comp_init(Eclock, x2i, i2x, & seq_flds_x2i_fields, seq_flds_i2x_fields, seq_flds_i2o_per_cat, & SDICE, gsmap, ggrid, mpicom, compid, my_task, master_task, & @@ -164,11 +170,6 @@ subroutine ice_init_mct( EClock, cdata, x2i, i2x, NLFilename ) #ifdef HAVE_MOAB - ierr = iMOAB_RegisterApplication(trim("DICE")//C_NULL_CHAR, mpicom, compid, MPSIID) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in registering data ice comp' - call shr_sys_abort(subname//' ERROR in registering data ice comp') - endif if (my_task == master_task) then call shr_stream_getDomainInfo(SDICE%stream(1), filePath,fileName,timeName,lonName, & latName,hgtName,maskName,areaName) From ef39f1f892ff61e73f6110bf2c8adb10476138eb Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sun, 16 Jun 2024 15:19:31 -0500 Subject: [PATCH 3/4] need to compute comm graph between data ice and coupler ice --- driver-moab/main/cplcomp_exchange_mod.F90 | 35 +++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/driver-moab/main/cplcomp_exchange_mod.F90 b/driver-moab/main/cplcomp_exchange_mod.F90 index e21d3f4554a2..a677153d4e67 100644 --- a/driver-moab/main/cplcomp_exchange_mod.F90 +++ b/driver-moab/main/cplcomp_exchange_mod.F90 @@ -1359,7 +1359,7 @@ subroutine cplcomp_moab_Init(infodata,comp) endif endif ! in case of domain read, we need to compute the comm graph - if ( trim(ocn_domain) /= 'none' ) then + if ( trim(ocn_domain) /= 'none' ) then ! we are now on joint pes, compute comm graph between data ocn and coupler model ocn typeA = 2 ! point cloud on component PEs typeB = 3 ! full mesh on coupler pes, we just read it @@ -1596,7 +1596,17 @@ subroutine cplcomp_moab_Init(infodata,comp) if (seq_comm_iamroot(CPLID)) then write(logunit,'(A)') subname//' load ice domain mesh from file '//trim(ice_domain) endif - +#ifdef MOABDEBUG + ! debug test + outfile = 'recSeaIceInit.h5m'//C_NULL_CHAR + wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ! + ! write out the mesh file to disk + ierr = iMOAB_WriteMesh(mbixid, trim(outfile), trim(wopts)) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in writing sea ice mesh on coupler ' + call shr_sys_abort(subname//' ERROR in writing sea ice mesh on coupler ') + endif +#endif endif tagtype = 1 ! dense, double numco = 1 ! one value per cell / entity @@ -1618,7 +1628,6 @@ subroutine cplcomp_moab_Init(infodata,comp) write(logunit,*) subname,' error in defining tags seq_flds_dom_fields on ice on coupler ' call shr_sys_abort(subname//' ERROR in defining tags ') endif - tagname = trim(seq_flds_a2x_fields)//C_NULL_CHAR tagtype = 1 ! dense numco = 1 ! @@ -1659,6 +1668,22 @@ subroutine cplcomp_moab_Init(infodata,comp) endif endif endif + ! in case of ice domain read, we need to compute the comm graph + if ( trim(ice_domain) /= 'none' ) then + ! we are now on joint pes, compute comm graph between data ice and coupler model ice + typeA = 2 ! point cloud on component PEs + typeB = 3 ! full mesh on coupler pes, we just read it + ierr = iMOAB_ComputeCommGraph( MPSIID, mbixid, mpicom_join, mpigrp_old, mpigrp_cplid, & + typeA, typeB, id_old, id_join) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in computing comm graph for data ice model ' + call shr_sys_abort(subname//' ERROR in computing comm graph for data ice model ') + endif + ! also, frac, area, aream, masks has to come from ice MPSIID , not from domain file reader + ! this is hard to digest :( + tagname = 'area:aream:frac:mask'//C_NULL_CHAR + call component_exch_moab(comp, MPSIID, mbixid, 0, tagname) + endif endif ! rof @@ -1810,14 +1835,14 @@ subroutine component_exch_moab(comp, mbAPPid1, mbAppid2, direction, fields ) ! basically, use the initial partitioning ierr = iMOAB_SendElementTag(mbAPPid1, tagName, mpicom_join, target_id) if (ierr .ne. 0) then - call shr_sys_abort(subname//' cannot send element tag') + call shr_sys_abort(subname//' cannot send element tag: '//trim(tagName)) endif endif if ( mbAPPid2 .ge. 0 ) then ! we are on receiving end ierr = iMOAB_ReceiveElementTag(mbAPPid2, tagName, mpicom_join, source_id) if (ierr .ne. 0) then - call shr_sys_abort(subname//' cannot receive element tag') + call shr_sys_abort(subname//' cannot receive element tag: '//trim(tagName)) endif endif From 53be196af1ccfff3e2a3b7c71e3c17bb1d2bb92a Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 18 Jun 2024 08:42:40 -0500 Subject: [PATCH 4/4] add GLOBAL_ID to tags for ice needed for restart, for all models that read a domain/mesh file this happens in cplcomp_exchange_mod this is for data ice, in which we read a domain file --- driver-moab/main/cplcomp_exchange_mod.F90 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/driver-moab/main/cplcomp_exchange_mod.F90 b/driver-moab/main/cplcomp_exchange_mod.F90 index a677153d4e67..76737ad31419 100644 --- a/driver-moab/main/cplcomp_exchange_mod.F90 +++ b/driver-moab/main/cplcomp_exchange_mod.F90 @@ -1596,6 +1596,15 @@ subroutine cplcomp_moab_Init(infodata,comp) if (seq_comm_iamroot(CPLID)) then write(logunit,'(A)') subname//' load ice domain mesh from file '//trim(ice_domain) endif + ! need to add global id tag to the app, it will be used in restart + tagtype = 0 ! dense, integer + numco = 1 + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mbixid, tagname, tagtype, numco, tagindex ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in adding global id tag to icex' + call shr_sys_abort(subname//' ERROR in adding global id tag to icex ') + endif #ifdef MOABDEBUG ! debug test outfile = 'recSeaIceInit.h5m'//C_NULL_CHAR