From 4e89f9312d95b80c763b4ca3b5263ba1df51e939 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Wed, 23 Oct 2024 10:05:13 -0600 Subject: [PATCH 1/4] initial working --- .../dynamics/mpas_atm_time_integration.F | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 6a8005c8b2..c512b30ef4 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -1893,26 +1893,37 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, integer :: iCell, k, iq real (kind=RKIND) :: dtseps, c2, qtotal, rcv real (kind=RKIND), dimension( nVertLevels ) :: b_tri, c_tri + !real (kind=RKIND), dimension( nVertLevels,nCells+1 ) :: b_tri, c_tri + MPAS_ACC_TIMER_START('atm_compute_vert_imp_coefs_work [ACC_data_xfer]') + !$acc enter data copyin(rdzw, zz, fzm, fzp, rdzu, cqw, p, t, qtot, rb, rtb, rt, pb) create(cofrz, cofwr, cofwz, coftz, cofwt, a_tri, b_tri, c_tri, alpha_tri, gamma_tri) + MPAS_ACC_TIMER_STOP('atm_compute_vert_imp_coefs_work [ACC_data_xfer]') ! set coefficients dtseps = .5*dts*(1.+epssm) rcv = rgas/(cp-rgas) c2 = cp*rcv + !$acc parallel + !$acc loop ! MGD bad to have all threads setting this variable? do k=1,nVertLevels cofrz(k) = dtseps*rdzw(k) end do - + !$acc end parallel + + !$acc parallel + !$acc loop do iCell = cellSolveStart,cellSolveEnd ! we only need to do cells we are solving for, not halo cells !DIR$ IVDEP + !$acc loop do k=2,nVertLevels cofwr(k,iCell) =.5*dtseps*gravity*(fzm(k)*zz(k,iCell)+fzp(k)*zz(k-1,iCell)) end do coftz(1,iCell) = 0.0 !DIR$ IVDEP + !$acc loop do k=2,nVertLevels cofwz(k,iCell) = dtseps*c2*(fzm(k)*zz(k,iCell)+fzp(k)*zz(k-1,iCell)) & *rdzu(k)*cqw(k,iCell)*(fzm(k)*p (k,iCell)+fzp(k)*p (k-1,iCell)) @@ -1920,6 +1931,7 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, end do coftz(nVertLevels+1,iCell) = 0.0 !DIR$ IVDEP + !$acc loop do k=1,nVertLevels ! qtotal = 0. @@ -1932,14 +1944,28 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, *p(k,iCell)/((rtb(k,iCell)+rt(k,iCell))*pb(k,iCell)) ! cofwt(k,iCell) = 0. end do + end do + !$acc end parallel + + !a_tri(1,iCell) = 0. ! note, this value is never used + b_tri(1) = 1. ! note, this value is never used + c_tri(1) = 0. ! note, this value is never used + + !$acc update device(b_tri,c_tri) + + !$acc parallel + !$acc loop gang worker private(b_tri,c_tri) + do iCell = cellSolveStart,cellSolveEnd ! we only need to do cells we are solving for, not halo cells + + a_tri(1,iCell) = 0. ! note, this value is never used + !b_tri(1,iCell) = 1. ! note, this value is never used + !c_tri(1,iCell) = 0. ! note, this value is never used + gamma_tri(1,iCell) = 0. + alpha_tri(1,iCell) = 0. ! note, this value is never used - a_tri(1,iCell) = 0. ! note, this value is never used - b_tri(1) = 1. ! note, this value is never used - c_tri(1) = 0. ! note, this value is never used - gamma_tri(1,iCell) = 0. - alpha_tri(1,iCell) = 0. ! note, this value is never used !DIR$ IVDEP + !$acc loop vector do k=2,nVertLevels a_tri(k,iCell) = -cofwz(k ,iCell)* coftz(k-1,iCell)*rdzw(k-1)*zz(k-1,iCell) & +cofwr(k ,iCell)* cofrz(k-1 ) & @@ -1954,13 +1980,18 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, -cofwr(k ,iCell)* cofrz(k ) & +cofwt(k ,iCell)* coftz(k+1,iCell)*rdzw(k ) end do + !MGD VECTOR DEPENDENCE + !$acc loop seq do k=2,nVertLevels alpha_tri(k,iCell) = 1./(b_tri(k)-a_tri(k,iCell)*gamma_tri(k-1,iCell)) gamma_tri(k,iCell) = c_tri(k)*alpha_tri(k,iCell) end do end do ! loop over cells + !$acc end parallel + + !$acc exit data copyout(cofrz, cofwr, cofwz, coftz, cofwt, a_tri, b_tri, c_tri, alpha_tri, gamma_tri) delete(rdzw, zz, fzm, fzp, rdzu, cqw, p, t, qtot, rb, rtb, rt, pb) end subroutine atm_compute_vert_imp_coefs_work From 2f4cf65c62cd088a08a01395897381acab0bda9e Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Wed, 23 Oct 2024 16:11:58 -0600 Subject: [PATCH 2/4] edits --- .../dynamics/mpas_atm_time_integration.F | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index c512b30ef4..909b744642 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -1904,7 +1904,7 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, rcv = rgas/(cp-rgas) c2 = cp*rcv - !$acc parallel + !$acc parallel !$acc loop ! MGD bad to have all threads setting this variable? do k=1,nVertLevels @@ -1912,8 +1912,8 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, end do !$acc end parallel - !$acc parallel - !$acc loop + !$acc parallel + !$acc loop gang worker private(b_tri,c_tri) do iCell = cellSolveStart,cellSolveEnd ! we only need to do cells we are solving for, not halo cells !DIR$ IVDEP @@ -1944,24 +1944,24 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, *p(k,iCell)/((rtb(k,iCell)+rt(k,iCell))*pb(k,iCell)) ! cofwt(k,iCell) = 0. end do - end do - !$acc end parallel + !end do + !!$acc end parallel - !a_tri(1,iCell) = 0. ! note, this value is never used - b_tri(1) = 1. ! note, this value is never used - c_tri(1) = 0. ! note, this value is never used + !a_tri(1,iCell) = 0. ! note, this value is never used + !b_tri(1) = 1. ! note, this value is never used + !c_tri(1) = 0. ! note, this value is never used - !$acc update device(b_tri,c_tri) + ! !$acc update device(b_tri,c_tri) - !$acc parallel - !$acc loop gang worker private(b_tri,c_tri) - do iCell = cellSolveStart,cellSolveEnd ! we only need to do cells we are solving for, not halo cells - - a_tri(1,iCell) = 0. ! note, this value is never used - !b_tri(1,iCell) = 1. ! note, this value is never used - !c_tri(1,iCell) = 0. ! note, this value is never used - gamma_tri(1,iCell) = 0. - alpha_tri(1,iCell) = 0. ! note, this value is never used + !!$acc parallel + !!$acc loop gang worker private(b_tri,c_tri) + !do iCell = cellSolveStart,cellSolveEnd ! we only need to do cells we are solving for, not halo cells + + a_tri(1,iCell) = 0. ! note, this value is never used + b_tri(1) = 1. ! note, this value is never used + c_tri(1) = 0. ! note, this value is never used + gamma_tri(1,iCell) = 0. + alpha_tri(1,iCell) = 0. ! note, this value is never used !DIR$ IVDEP From ed5560f95b3297adb1c8c6cecf536aeb28daf270 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Wed, 23 Oct 2024 16:30:44 -0600 Subject: [PATCH 3/4] changes to formatting --- .../dynamics/mpas_atm_time_integration.F | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 909b744642..d88959e43d 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -1893,10 +1893,12 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, integer :: iCell, k, iq real (kind=RKIND) :: dtseps, c2, qtotal, rcv real (kind=RKIND), dimension( nVertLevels ) :: b_tri, c_tri - !real (kind=RKIND), dimension( nVertLevels,nCells+1 ) :: b_tri, c_tri MPAS_ACC_TIMER_START('atm_compute_vert_imp_coefs_work [ACC_data_xfer]') - !$acc enter data copyin(rdzw, zz, fzm, fzp, rdzu, cqw, p, t, qtot, rb, rtb, rt, pb) create(cofrz, cofwr, cofwz, coftz, cofwt, a_tri, b_tri, c_tri, alpha_tri, gamma_tri) + !$acc enter data copyin(rdzw, zz, fzm, fzp, rdzu, cqw, p, t, qtot, rb, & + !$acc rtb, rt, pb) + !$acc enter data create(cofrz, cofwr, cofwz, coftz, cofwt, a_tri, b_tri, & + !$acc c_tri, alpha_tri, gamma_tri) MPAS_ACC_TIMER_STOP('atm_compute_vert_imp_coefs_work [ACC_data_xfer]') ! set coefficients @@ -1944,18 +1946,6 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, *p(k,iCell)/((rtb(k,iCell)+rt(k,iCell))*pb(k,iCell)) ! cofwt(k,iCell) = 0. end do - !end do - !!$acc end parallel - - !a_tri(1,iCell) = 0. ! note, this value is never used - !b_tri(1) = 1. ! note, this value is never used - !c_tri(1) = 0. ! note, this value is never used - - ! !$acc update device(b_tri,c_tri) - - !!$acc parallel - !!$acc loop gang worker private(b_tri,c_tri) - !do iCell = cellSolveStart,cellSolveEnd ! we only need to do cells we are solving for, not halo cells a_tri(1,iCell) = 0. ! note, this value is never used b_tri(1) = 1. ! note, this value is never used @@ -1963,7 +1953,6 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, gamma_tri(1,iCell) = 0. alpha_tri(1,iCell) = 0. ! note, this value is never used - !DIR$ IVDEP !$acc loop vector do k=2,nVertLevels @@ -1991,7 +1980,9 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, end do ! loop over cells !$acc end parallel - !$acc exit data copyout(cofrz, cofwr, cofwz, coftz, cofwt, a_tri, b_tri, c_tri, alpha_tri, gamma_tri) delete(rdzw, zz, fzm, fzp, rdzu, cqw, p, t, qtot, rb, rtb, rt, pb) + !$acc exit data copyout(cofrz, cofwr, cofwz, coftz, cofwt, a_tri, b_tri, & + !$acc c_tri, alpha_tri, gamma_tri) + !$acc exit data delete(rdzw, zz, fzm, fzp, rdzu, cqw, p, t, qtot, rb, rtb, rt, pb) end subroutine atm_compute_vert_imp_coefs_work From bd7bbdde02c14353cbb4b6020d515a894eb8a891 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Wed, 23 Oct 2024 16:34:04 -0600 Subject: [PATCH 4/4] removing whitespace --- src/core_atmosphere/dynamics/mpas_atm_time_integration.F | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index d88959e43d..1fda8c4da3 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -1969,7 +1969,6 @@ subroutine atm_compute_vert_imp_coefs_work(nCells, moist_start, moist_end, dts, -cofwr(k ,iCell)* cofrz(k ) & +cofwt(k ,iCell)* coftz(k+1,iCell)*rdzw(k ) end do - !MGD VECTOR DEPENDENCE !$acc loop seq do k=2,nVertLevels