Skip to content

Commit

Permalink
Merge pull request #13 from NOAA-EMC/feature/perf
Browse files Browse the repository at this point in the history 
Enable new decomposition algorithm
  • Loading branch information
@barlage
barlage authored Jan 28, 2025
2 parents 025043d + 2b832ca commit 1a2d2fc
Show file tree
Hide file tree
Showing 7 changed files with 333 additions and 32 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/datm_noahmp.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ on:

jobs:
set-matrix:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04

outputs:
matrix: ${{ steps.generate.outputs.matrix }}
Expand Down Expand Up @@ -88,7 +88,7 @@ jobs:
build:
needs: set-matrix

runs-on: ubuntu-latest
runs-on: ubuntu-24.04

strategy:
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
Expand Down
35 changes: 25 additions & 10 deletions drivers/ccpp/update.sh
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,39 @@
#!/bin/bash

cp -f ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 .
echo "checksum - before"
md5sum ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 funcphys.f90
cp -f ../../../../FV3/ccpp/physics/physics/hooks/machine.F .
md5sum ../../../../FV3/ccpp/physics/physics/hooks/machine.F machine.F
cp -f ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 .
md5sum ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 physcons.F90
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f namelist_soilveg.f
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f set_soilveg.f
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f sfc_diff.f
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 noahmp_tables.f90
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 noahmpdrv.F90
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta noahmpdrv.meta
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/.
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/module_sf_noahmp_glacier.F90
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmplsm.F90 ../../src/module_sf_noahmplsm.F90

cp -f ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 .
cp -f ../../../../FV3/ccpp/physics/physics/hooks/machine.F .
cp -f ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/.
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmplsm.F90 ../../src/.

echo "checksum - after"
md5sum ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 funcphys.f90
md5sum ../../../../FV3/ccpp/physics/physics/hooks/machine.F machine.F
md5sum ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 physcons.F90
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f namelist_soilveg.f
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f set_soilveg.f
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f sfc_diff.f
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 noahmp_tables.f90
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 noahmpdrv.F90
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta noahmpdrv.meta
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/module_sf_noahmp_glacier.F90
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmplsm.F90 ../../src/module_sf_noahmplsm.F90
252 changes: 249 additions & 3 deletions drivers/nuopc/lnd_comp_domain.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ module lnd_comp_domain
use ESMF , only : ESMF_RouteHandleDestroy, ESMF_GridGet, ESMF_GridGetCoord
use ESMF , only : ESMF_FieldRegridGetArea, ESMF_CoordSys_Flag
use ESMF , only : ESMF_MeshGetFieldBounds, ESMF_COORDSYS_CART, ESMF_KIND_R8
use ESMF , only : ESMF_MeshDestroy, ESMF_DistGridCreate, ESMF_VMAllGatherV
use NUOPC, only : NUOPC_CompAttributeGet

use lnd_comp_kind , only : r4 => shr_kind_r4
Expand Down Expand Up @@ -62,10 +63,11 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)
! local variables
real(r4), target, allocatable :: tmpr4(:)
real(r8), target, allocatable :: tmpr8(:)
integer :: n
integer :: n, petCount, localPet
integer :: decomptile(2,6)
integer :: maxIndex(2)
type(ESMF_Decomp_Flag) :: decompflagPTile(2,6)
type(ESMF_VM) :: vm

type(field_type) :: flds(1)
integer :: numOwnedElements, spatialDim, rank
Expand All @@ -87,6 +89,16 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)
rc = ESMF_SUCCESS
call ESMF_LogWrite(subname//' called', ESMF_LOGMSG_INFO)

! ---------------------
! Query VM
! ---------------------

call ESMF_GridCompGet(gcomp, vm=vm, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

call ESMF_VMGet(vm, petCount=petCount, localPet=localPet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

! ---------------------
! Set decomposition and decide it is regional or global
! ---------------------
Expand All @@ -101,7 +113,14 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)
! set number of tiles
noahmp%domain%ntiles = 6

! set decomposition
! check user provided layout
if (petCount /= noahmp%domain%layout(1)*noahmp%domain%layout(2)*noahmp%domain%ntiles) then
call ESMF_LogWrite(trim(subname)//": ERROR in layout. layout_x * layout_y * 6 != #PETs", ESMF_LOGMSG_INFO)
rc = ESMF_FAILURE
return
end if

! use user provided layout to set decomposition
do n = 1, noahmp%domain%ntiles
decomptile(1,n) = noahmp%domain%layout(1)
decomptile(2,n) = noahmp%domain%layout(2)
Expand Down Expand Up @@ -226,7 +245,7 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
vegtype(:) = int(tmpr8)
else
write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%input_dir)//'C',noahmp%domain%ni, '.vegetation_type.tile*.nc'
write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%fixed_dir)//'C',noahmp%domain%ni, '.vegetation_type.tile*.nc'
flds(1)%short_name = 'vegetation_type'
flds(1)%ptr1r4 => tmpr4
call read_tiled_file(noahmp, filename, flds, rc=rc)
Expand Down Expand Up @@ -257,6 +276,16 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)
call ESMF_MeshSet(noahmp%domain%mesh, elementMask=noahmp%domain%mask, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! ---------------------
! Modify decomposition to evenly distribute land and ocean points
! ---------------------

if (trim(noahmp%nmlist%decomp_type) == 'custom') then
! modify decomposition
call lnd_modify_decomp(gcomp, noahmp, rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
end if

! ---------------------
! Get height from orography file
! ---------------------
Expand Down Expand Up @@ -339,4 +368,221 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)

end subroutine lnd_set_decomp_and_domain_from_mosaic

!===============================================================================
subroutine lnd_modify_decomp(gcomp, noahmp, rc)

! input/output variables
type(ESMF_GridComp), intent(in) :: gcomp
type(noahmp_type), intent(inout) :: noahmp
integer, intent(out) :: rc

! local variables
type(ESMF_VM) :: vm
type(ESMF_Mesh) :: mesh
type(ESMF_DistGrid) :: distgrid, distgrid_new
type(field_type) :: flds(1)
integer :: n, m, g
integer :: petCount, localPet
integer :: lsize, gsize(1)
integer :: nlnd, nocn
integer :: begl_l, endl_l, begl_o, endl_o
integer, allocatable :: nlnd_loc(:)
integer, allocatable :: nocn_loc(:)
integer, allocatable :: mask_glb(:)
integer, allocatable :: gindex_loc(:)
integer, allocatable :: gindex_glb(:)
integer, allocatable :: gindex_lnd(:)
integer, allocatable :: gindex_ocn(:)
integer, allocatable :: gindex_new(:)
integer, allocatable :: lsize_arr(:)
real(r4), target, allocatable :: tmpr4(:)
character(len=cl) :: msg, filename
character(len=*), parameter :: subname = trim(modName)//':(lnd_modify_decomp) '
!-------------------------------------------------------------------------------

rc = ESMF_SUCCESS
call ESMF_LogWrite(subname//' called', ESMF_LOGMSG_INFO)

! ---------------------
! Query VM
! ---------------------

call ESMF_GridCompGet(gcomp, vm=vm, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

call ESMF_VMGet(vm, petCount=petCount, localPet=localPet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

! ---------------------
! Query existing mesh
! ---------------------

! retrive default distgrid
call ESMF_MeshGet(noahmp%domain%mesh, elementdistGrid=distgrid, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! get local number of elements
call ESMF_DistGridGet(distgrid, localDe=0, elementCount=lsize, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! calculate number of elements globally
call ESMF_VMAllReduce(vm, (/ lsize /), gsize, 1, ESMF_REDUCE_SUM, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

nlnd = count(noahmp%domain%mask(:) > 0, dim=1)
nocn = lsize-nlnd
write(msg, fmt='(A,4I8)') trim(subname)//' : lsize, gsize, nlnd, nocn = ', &
lsize, gsize(1), nlnd, nocn
call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO)

! get default sequence index
allocate(gindex_loc(lsize))
call ESMF_DistGridGet(distgrid, 0, seqIndexList=gindex_loc, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! ---------------------
! Create global view of indexes and mask
! --------------------

! collect local sizes
allocate(lsize_arr(petCount))
call ESMF_VMAllGatherV(vm, sendData=(/ lsize /), sendCount=1, &
recvData=lsize_arr, recvCounts=(/ (1, n = 0, petCount-1) /), &
recvOffsets=(/ (n, n = 0, petCount-1) /), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! global view of indexes
allocate(gindex_glb(gsize(1)))
gindex_glb(:) = 0
call ESMF_VMAllGatherV(vm, sendData=gindex_loc, sendCount=lsize, &
recvData=gindex_glb, recvCounts=lsize_arr, &
recvOffsets=(/ (sum(lsize_arr(1:n))-lsize_arr(1), n = 1, petCount) /), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! global view of land sea mask
allocate(mask_glb(gsize(1)))
mask_glb(:) = 0
call ESMF_VMAllGatherV(vm, sendData=noahmp%domain%mask, sendCount=lsize, &
recvData=mask_glb, recvCounts=lsize_arr, &
recvOffsets=(/ (sum(lsize_arr(1:n))-lsize_arr(1), n = 1, petCount) /), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! ---------------------
! split global indexes as land and ocean
! ---------------------

nlnd = count(mask_glb > 0, dim=1)
nocn = gsize(1)-nlnd
allocate(gindex_lnd(nlnd))
gindex_lnd = 0
allocate(gindex_ocn(nocn))
gindex_ocn = 0

n = 0
m = 0
do g = 1, gsize(1)
if (mask_glb(g) > 0) then
n = n+1
gindex_lnd(n) = gindex_glb(g)
else
m = m+1
gindex_ocn(m) = gindex_glb(g)
end if
end do

! ---------------------
! create new local indexes
! ---------------------

allocate(nlnd_loc(0:petCount-1))
nlnd_loc = 0
allocate(nocn_loc(0:petCount-1))
nocn_loc = 0
do n = 0, petCount-1
nlnd_loc(n) = nlnd/petCount
nocn_loc(n) = nocn/petCount
if (n < mod(nlnd, petCount)) then
nlnd_loc(n) = nlnd_loc(n)+1
else
nocn_loc(n) = nocn_loc(n)+1
end if
end do
if (localPet == 0) then
begl_l = 1
begl_o = 1
else
begl_l = sum(nlnd_loc(0:localPet-1))+1
begl_o = sum(nocn_loc(0:localPet-1))+1
end if
endl_l = sum(nlnd_loc(0:localPet))
endl_o = sum(nocn_loc(0:localPet))

write(msg,'(A,6I8)') trim(subname)//' : nlnd_loc, nocn_loc, begl_l, endl_l, begl_o, endl_o = ', &
nlnd_loc(localPet), nocn_loc(localPet), begl_l, endl_l, begl_o, endl_o
call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO)

allocate(gindex_new(nlnd_loc(localPet)+nocn_loc(localPet)))
gindex_new(:nlnd_loc(localPet)) = gindex_lnd(begl_l:begl_l)
gindex_new(nlnd_loc(localPet)+1:) = gindex_ocn(begl_o:endl_o)

if (noahmp%nmlist%debug_level > 10) then
do n = 1, nlnd_loc(localPet)+nocn_loc(localPet)
write(msg,'(A,2I8)') trim(subname)//' : n, gindex_new = ', n, gindex_new(n)
call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO)
end do
end if

! ---------------------
! Update mesh with new decomposition
! ---------------------

! create new distgrid with new index
distgrid_new = ESMF_DistGridCreate(arbSeqIndexList=gindex_new, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! create new mesh with new distgrid
mesh = ESMF_MeshCreate(noahmp%domain%mesh, elementDistGrid=distgrid_new, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! destroy old and replace with new one
call ESMF_MeshDestroy(noahmp%domain%mesh, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
noahmp%domain%mesh = mesh

! ---------------------
! fix mask and fraction to be consistent with new decomposition
! ---------------------

! mask
noahmp%domain%mask(:nlnd_loc(localPet)) = 1
noahmp%domain%mask(nlnd_loc(localPet)+1:) = 0

! fraction, read from file again
allocate(tmpr4(noahmp%domain%begl:noahmp%domain%endl))
tmpr4(:) = 0.0
filename = trim(noahmp%nmlist%input_dir)//'oro_data.tile*.nc'
flds(1)%short_name = 'land_frac'
flds(1)%ptr1r4 => tmpr4
call read_tiled_file(noahmp, filename, flds, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
deallocate(tmpr4)

! ---------------------
! Clean memory
! ---------------------

deallocate(nlnd_loc)
deallocate(nocn_loc)
deallocate(mask_glb)
deallocate(gindex_loc)
deallocate(gindex_glb)
deallocate(gindex_lnd)
deallocate(gindex_ocn)
deallocate(gindex_new)
deallocate(lsize_arr)

call ESMF_LogWrite(subname//' done', ESMF_LOGMSG_INFO)

end subroutine lnd_modify_decomp

end module lnd_comp_domain
Loading

0 comments on commit 1a2d2fc

Please sign in to comment.