Skip to content

Commit

Permalink
Removal of mpas_get_array_gpu calls
Browse files Browse the repository at this point in the history 
Replaced the mpas_pool_get_array_gpu function by mpas_pool_get_array
for the routines solve_diagnostics and init_coupled_diagnostics.
The _gpu calls has !$acc data copyin directives and it transfers the
variables on to gpu. The two functions solve_diagnostics and init_coupled_
diagnostics functions are also called outside the time integration. The
data copying is corrupting the GPU data later in the timestep.
This results in NaNs.
  • Loading branch information
@supreethms1809
supreethms1809 committed Apr 6, 2018
1 parent 70ff119 commit 67608e0
Showing 1 changed file with 77 additions and 79 deletions.
156 changes: 77 additions & 79 deletions src/core_atmosphere/dynamics/mpas_atm_time_integration.F
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ subroutine atm_timestep(domain, dt, timeStamp, itimestep)
block => domain % blocklist
do while (associated(block))
call mpas_pool_get_subpool(block % structs, 'state', state)
call mpas_pool_get_array_gpu(state, 'xtime', xtime, 2)
call mpas_pool_get_array(state, 'xtime', xtime, 2)
xtime = xtime_new
block => block % next
end do
Expand Down Expand Up @@ -6500,10 +6500,8 @@ subroutine atm_compute_solve_diagnostics_work_gpu(nCells, nEdges, nVertices, &
v(k,iEdge) = 0.0
do i=1,nEdgesOnEdge(iEdge)
eoe = edgesOnEdge(i,iEdge)
! temp = temp + weightsOnEdge(i,iEdge) * u(k, eoe)
v(k,iEdge) = v(k,iEdge) + weightsOnEdge(i,iEdge) * u(k, eoe)
end do
! v(k,iEdge) = temp
end do
end do
!$acc end parallel
Expand Down Expand Up @@ -6650,42 +6648,42 @@ subroutine atm_compute_solve_diagnostics(dt, state, time_lev, diag, mesh, config

call mpas_pool_get_config(configs, 'config_apvm_upwinding', config_apvm_upwinding)

call mpas_pool_get_array_gpu(state, 'rho_zz', h, time_lev)
call mpas_pool_get_array_gpu(state, 'u', u, time_lev)

call mpas_pool_get_array_gpu(diag, 'v', v)
call mpas_pool_get_array_gpu(diag, 'rho_edge', h_edge)
call mpas_pool_get_array_gpu(diag, 'vorticity', vorticity)
call mpas_pool_get_array_gpu(diag, 'divergence', divergence)
call mpas_pool_get_array_gpu(diag, 'ke', ke)
call mpas_pool_get_array_gpu(diag, 'pv_edge', pv_edge)
call mpas_pool_get_array_gpu(diag, 'pv_vertex', pv_vertex)
call mpas_pool_get_array_gpu(diag, 'pv_cell', pv_cell)
call mpas_pool_get_array_gpu(diag, 'gradPVn', gradPVn)
call mpas_pool_get_array_gpu(diag, 'gradPVt', gradPVt)

call mpas_pool_get_array_gpu(mesh, 'weightsOnEdge', weightsOnEdge)
call mpas_pool_get_array_gpu(mesh, 'kiteAreasOnVertex', kiteAreasOnVertex)
call mpas_pool_get_array_gpu(mesh, 'cellsOnEdge', cellsOnEdge)
call mpas_pool_get_array_gpu(mesh, 'cellsOnVertex', cellsOnVertex)
call mpas_pool_get_array_gpu(mesh, 'verticesOnEdge', verticesOnEdge)
call mpas_pool_get_array_gpu(mesh, 'verticesOnCell', verticesOnCell)
call mpas_pool_get_array_gpu(mesh, 'nEdgesOnCell', nEdgesOnCell)
call mpas_pool_get_array_gpu(mesh, 'edgesOnCell', edgesOnCell)
call mpas_pool_get_array_gpu(mesh, 'nEdgesOnEdge', nEdgesOnEdge)
call mpas_pool_get_array_gpu(mesh, 'edgesOnEdge', edgesOnEdge)
call mpas_pool_get_array_gpu(mesh, 'edgesOnVertex', edgesOnVertex)
call mpas_pool_get_array_gpu(mesh, 'edgesOnVertex_sign', edgesOnVertex_sign)
call mpas_pool_get_array_gpu(mesh, 'edgesOnCell_sign', edgesOnCell_sign)
call mpas_pool_get_array_gpu(mesh, 'kiteForCell', kiteForCell)
call mpas_pool_get_array_gpu(mesh, 'dcEdge', dcEdge)
call mpas_pool_get_array_gpu(mesh, 'dvEdge', dvEdge)
call mpas_pool_get_array_gpu(mesh, 'invDcEdge', invDcEdge)
call mpas_pool_get_array_gpu(mesh, 'invDvEdge', invDvEdge)
call mpas_pool_get_array_gpu(mesh, 'invAreaCell', invAreaCell)
call mpas_pool_get_array_gpu(mesh, 'invAreaTriangle', invAreaTriangle)
call mpas_pool_get_array_gpu(mesh, 'fVertex', fVertex)
call mpas_pool_get_array_gpu(mesh, 'fEdge', fEdge)
call mpas_pool_get_array(state, 'rho_zz', h, time_lev)
call mpas_pool_get_array(state, 'u', u, time_lev)

call mpas_pool_get_array(diag, 'v', v)
call mpas_pool_get_array(diag, 'rho_edge', h_edge)
call mpas_pool_get_array(diag, 'vorticity', vorticity)
call mpas_pool_get_array(diag, 'divergence', divergence)
call mpas_pool_get_array(diag, 'ke', ke)
call mpas_pool_get_array(diag, 'pv_edge', pv_edge)
call mpas_pool_get_array(diag, 'pv_vertex', pv_vertex)
call mpas_pool_get_array(diag, 'pv_cell', pv_cell)
call mpas_pool_get_array(diag, 'gradPVn', gradPVn)
call mpas_pool_get_array(diag, 'gradPVt', gradPVt)

call mpas_pool_get_array(mesh, 'weightsOnEdge', weightsOnEdge)
call mpas_pool_get_array(mesh, 'kiteAreasOnVertex', kiteAreasOnVertex)
call mpas_pool_get_array(mesh, 'cellsOnEdge', cellsOnEdge)
call mpas_pool_get_array(mesh, 'cellsOnVertex', cellsOnVertex)
call mpas_pool_get_array(mesh, 'verticesOnEdge', verticesOnEdge)
call mpas_pool_get_array(mesh, 'verticesOnCell', verticesOnCell)
call mpas_pool_get_array(mesh, 'nEdgesOnCell', nEdgesOnCell)
call mpas_pool_get_array(mesh, 'edgesOnCell', edgesOnCell)
call mpas_pool_get_array(mesh, 'nEdgesOnEdge', nEdgesOnEdge)
call mpas_pool_get_array(mesh, 'edgesOnEdge', edgesOnEdge)
call mpas_pool_get_array(mesh, 'edgesOnVertex', edgesOnVertex)
call mpas_pool_get_array(mesh, 'edgesOnVertex_sign', edgesOnVertex_sign)
call mpas_pool_get_array(mesh, 'edgesOnCell_sign', edgesOnCell_sign)
call mpas_pool_get_array(mesh, 'kiteForCell', kiteForCell)
call mpas_pool_get_array(mesh, 'dcEdge', dcEdge)
call mpas_pool_get_array(mesh, 'dvEdge', dvEdge)
call mpas_pool_get_array(mesh, 'invDcEdge', invDcEdge)
call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge)
call mpas_pool_get_array(mesh, 'invAreaCell', invAreaCell)
call mpas_pool_get_array(mesh, 'invAreaTriangle', invAreaTriangle)
call mpas_pool_get_array(mesh, 'fVertex', fVertex)
call mpas_pool_get_array(mesh, 'fEdge', fEdge)

call mpas_pool_get_dimension(mesh, 'nCells', nCells)
call mpas_pool_get_dimension(mesh, 'nEdges', nEdges)
Expand Down Expand Up @@ -7065,35 +7063,35 @@ subroutine atm_init_coupled_diagnostics(state, time_lev, diag, mesh, configs, &
call mpas_pool_get_dimension(mesh, 'nVertLevels', nVertLevels)
call mpas_pool_get_dimension(state, 'index_qv', index_qv)
call mpas_pool_get_array_gpu(mesh, 'cellsOnEdge', cellsOnEdge)
call mpas_pool_get_array_gpu(mesh, 'nEdgesOnCell', nEdgesOnCell)
call mpas_pool_get_array_gpu(mesh, 'edgesOnCell', edgesOnCell)
call mpas_pool_get_array_gpu(mesh, 'edgesOnCell_sign', edgesOnCell_sign)
call mpas_pool_get_array_gpu(state, 'theta_m', theta_m, time_lev)
call mpas_pool_get_array_gpu(diag, 'theta', theta)
call mpas_pool_get_array_gpu(state, 'rho_zz', rho_zz, time_lev)
call mpas_pool_get_array_gpu(diag, 'rho', rho)
call mpas_pool_get_array_gpu(diag, 'rho_p', rho_p)
call mpas_pool_get_array_gpu(diag, 'rho_base', rho_base)
call mpas_pool_get_array_gpu(diag, 'rtheta_base', rtheta_base)
call mpas_pool_get_array_gpu(diag, 'theta_base', theta_base)
call mpas_pool_get_array_gpu(diag, 'rtheta_p', rtheta_p)
call mpas_pool_get_array_gpu(mesh, 'zz', zz)
call mpas_pool_get_array_gpu(state, 'scalars', scalars, time_lev)
call mpas_pool_get_array_gpu(diag, 'ru', ru)
call mpas_pool_get_array_gpu(diag, 'rw', rw)
call mpas_pool_get_array_gpu(state, 'u', u, time_lev)
call mpas_pool_get_array_gpu(state, 'w', w, time_lev)
call mpas_pool_get_array_gpu(diag, 'pressure_p', pressure_p)
call mpas_pool_get_array_gpu(diag, 'exner', exner)
call mpas_pool_get_array_gpu(diag, 'exner_base', exner_base)
call mpas_pool_get_array_gpu(mesh, 'fzm', fzm)
call mpas_pool_get_array_gpu(mesh, 'fzp', fzp)
call mpas_pool_get_array_gpu(mesh, 'zb', zb)
call mpas_pool_get_array_gpu(mesh, 'zb3', zb3)
call mpas_pool_get_array_gpu(mesh, 'zb_cell', zb_cell)
call mpas_pool_get_array_gpu(mesh, 'zb3_cell', zb3_cell)
call mpas_pool_get_array(mesh, 'cellsOnEdge', cellsOnEdge)
call mpas_pool_get_array(mesh, 'nEdgesOnCell', nEdgesOnCell)
call mpas_pool_get_array(mesh, 'edgesOnCell', edgesOnCell)
call mpas_pool_get_array(mesh, 'edgesOnCell_sign', edgesOnCell_sign)
call mpas_pool_get_array(state, 'theta_m', theta_m, time_lev)
call mpas_pool_get_array(diag, 'theta', theta)
call mpas_pool_get_array(state, 'rho_zz', rho_zz, time_lev)
call mpas_pool_get_array(diag, 'rho', rho)
call mpas_pool_get_array(diag, 'rho_p', rho_p)
call mpas_pool_get_array(diag, 'rho_base', rho_base)
call mpas_pool_get_array(diag, 'rtheta_base', rtheta_base)
call mpas_pool_get_array(diag, 'theta_base', theta_base)
call mpas_pool_get_array(diag, 'rtheta_p', rtheta_p)
call mpas_pool_get_array(mesh, 'zz', zz)
call mpas_pool_get_array(state, 'scalars', scalars, time_lev)
call mpas_pool_get_array(diag, 'ru', ru)
call mpas_pool_get_array(diag, 'rw', rw)
call mpas_pool_get_array(state, 'u', u, time_lev)
call mpas_pool_get_array(state, 'w', w, time_lev)
call mpas_pool_get_array(diag, 'pressure_p', pressure_p)
call mpas_pool_get_array(diag, 'exner', exner)
call mpas_pool_get_array(diag, 'exner_base', exner_base)
call mpas_pool_get_array(mesh, 'fzm', fzm)
call mpas_pool_get_array(mesh, 'fzp', fzp)
call mpas_pool_get_array(mesh, 'zb', zb)
call mpas_pool_get_array(mesh, 'zb3', zb3)
call mpas_pool_get_array(mesh, 'zb_cell', zb_cell)
call mpas_pool_get_array(mesh, 'zb3_cell', zb3_cell)
rcv = rgas / (cp-rgas)
Expand Down Expand Up @@ -7444,14 +7442,14 @@ subroutine summarize_timestep(domain)
call mpas_pool_get_subpool(block % structs, 'diag', diag)
call mpas_pool_get_subpool(block % structs, 'mesh', mesh)
call mpas_pool_get_array_gpu(state, 'w', w, 2)
call mpas_pool_get_array_gpu(state, 'u', u, 2)
call mpas_pool_get_array_gpu(diag, 'v', v)
call mpas_pool_get_array_gpu(mesh, 'indexToCellID', indexToCellID)
call mpas_pool_get_array_gpu(mesh, 'latCell', latCell)
call mpas_pool_get_array_gpu(mesh, 'lonCell', lonCell)
call mpas_pool_get_array_gpu(mesh, 'latEdge', latEdge)
call mpas_pool_get_array_gpu(mesh, 'lonEdge', lonEdge)
call mpas_pool_get_array(state, 'w', w, 2)
call mpas_pool_get_array(state, 'u', u, 2)
call mpas_pool_get_array(diag, 'v', v)
call mpas_pool_get_array(mesh, 'indexToCellID', indexToCellID)
call mpas_pool_get_array(mesh, 'latCell', latCell)
call mpas_pool_get_array(mesh, 'lonCell', lonCell)
call mpas_pool_get_array(mesh, 'latEdge', latEdge)
call mpas_pool_get_array(mesh, 'lonEdge', lonEdge)
call mpas_pool_get_dimension(state, 'nCellsSolve', nCellsSolve)
call mpas_pool_get_dimension(state, 'nEdgesSolve', nEdgesSolve)
call mpas_pool_get_dimension(state, 'nVertLevels', nVertLevels)
Expand Down Expand Up @@ -7666,8 +7664,8 @@ subroutine summarize_timestep(domain)
do while (associated(block))
call mpas_pool_get_subpool(block % structs, 'state', state)
call mpas_pool_get_array_gpu(state, 'w', w, 2)
call mpas_pool_get_array_gpu(state, 'u', u, 2)
call mpas_pool_get_array(state, 'w', w, 2)
call mpas_pool_get_array(state, 'u', u, 2)
call mpas_pool_get_dimension(state, 'nCellsSolve', nCellsSolve)
call mpas_pool_get_dimension(state, 'nEdgesSolve', nEdgesSolve)
call mpas_pool_get_dimension(state, 'nVertLevels', nVertLevels)
Expand Down Expand Up @@ -7709,7 +7707,7 @@ subroutine summarize_timestep(domain)
do while (associated(block))
call mpas_pool_get_subpool(block % structs, 'state', state)
call mpas_pool_get_array_gpu(state, 'scalars', scalars, 2)
call mpas_pool_get_array(state, 'scalars', scalars, 2)
call mpas_pool_get_dimension(state, 'nCellsSolve', nCellsSolve)
call mpas_pool_get_dimension(state, 'nVertLevels', nVertLevels)
call mpas_pool_get_dimension(state, 'num_scalars', num_scalars)
Expand Down

0 comments on commit 67608e0

Please sign in to comment.