From fb9f81d4dc9f1cc1f6c60102db464bbe6fde7a51 Mon Sep 17 00:00:00 2001 From: Bob Robey Date: Fri, 13 May 2022 18:44:00 -0600 Subject: [PATCH 1/6] Modes for second if block of splitExplicitStep --- .../mpas_ocn_time_integration_split.F | 179 +++++++++++++++++- 1 file changed, 171 insertions(+), 8 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index 37a07fcc07a8..e28e63fb8de0 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -1787,9 +1787,51 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ elseif (splitExplicitStep == numTSIterations) then +#ifdef MPAS_OPENACC + !$acc enter data copyin(layerThicknessTend, layerThicknessNew, layerThicknessCur) + !$acc enter data copyin(activeTracersNew) + !$acc enter data copyin(normalVelocityNew) + !$acc enter data copyin(normalBarotropicVelocityNew) + !$acc enter data copyin(normalBaroclinicVelocityNew) + !$acc enter data copyin(normalBaroclinicVelocityCur) + if (varinp_compute_active_tracer_budgets) then + !$acc enter data copyin(layerThickEdgeFlux, activetracerhorizontaladvectionedgeflux) + !$acc enter data copyin(activeTracerHorizontalAdvectionTendency, activeTracerHorMixTendency, activeTracerVerticalAdvectionTendency, & + !$acc activeTracerSurfaceFluxTendency, activeTracerNonLocalTendency, temperatureShortWaveTendency) + endif + if (config_use_freq_filtered_thickness) then + !$acc enter data copyin(lowFreqDivergenceNew, lowFreqDivergenceCur, lowFreqDivergenceTend) + endif +#endif + + +#ifdef MPAS_OPENACC + !$acc parallel present(minLevelCell, maxLevelCell) & + !$acc present(layerThicknessTend) & + !$acc present(layerThicknessCur) & + !$acc present(minLevelEdgeBot, maxLevelEdgeTop) & + !$acc present(layerThickEdgeFlux) & + !$acc present(activeTracerHorizontalAdvectionEdgeFlux) & + !$acc present(layerThicknessNew) & + !$acc present(activeTracerHorizontalAdvectionTendency) & + !$acc present(activeTracerVerticalAdvectionTendency) & + !$acc present(activeTracerHorMixTendency) & + !$acc present(activeTracerSurfaceFluxTendency) & + !$acc present(temperatureShortWaveTendency) & + !$acc present(activeTracerNonLocalTendency) +#endif + + +#ifdef MPAS_OPENACC + !$acc loop gang +#else !$omp parallel !$omp do schedule(runtime) private(k) +#endif do iCell = 1, nCellsAll +#ifdef MPAS_OPENACC + !$acc loop vector +#endif do k = minLevelCell(iCell), maxLevelCell(iCell) ! this is h_{n+1} layerThicknessNew(k,iCell) = & @@ -1802,22 +1844,64 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ !$omp end parallel #endif - if (config_compute_active_tracer_budgets) then + if (varinp_compute_active_tracer_budgets) then +#ifdef MPAS_OPENACC + !$acc loop gang +#else !$omp parallel !$omp do schedule(runtime) private(k) +#endif do iEdge = 1, nEdgesAll +#ifdef MPAS_OPENACC + !$acc loop seq +#endif do k= minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) - activeTracerHorizontalAdvectionEdgeFlux(:,k,iEdge) = & - activeTracerHorizontalAdvectionEdgeFlux(:,k,iEdge) / & - layerThickEdgeFlux(k,iEdge) +#ifdef MPAS_OPENACC + !$acc loop vector +#endif + do iTracer = 1, size(activeTracerHorizontalAdvectionEdgeFlux,1) + activeTracerHorizontalAdvectionEdgeFlux(iTracer,k,iEdge) = & + activeTracerHorizontalAdvectionEdgeFlux(iTracer,k,iEdge) / & + layerThickEdgeFlux(k,iEdge) + enddo enddo enddo +#ifndef MPAS_OPENACC !$omp end do +#endif +#ifdef MPAS_OPENACC + !$acc loop gang +#else !$omp do schedule(runtime) private(k) +#endif do iCell = 1, nCellsAll do k= minLevelCell(iCell), maxLevelCell(iCell) +#ifdef MPAS_OPENACC + !$acc loop vector + do iTracer = 1, size(activeTracerHorizontalAdvectionTendency,1) + activeTracerHorizontalAdvectionTendency(iTracer,k,iCell) = & + activeTracerHorizontalAdvectionTendency(iTracer,k,iCell) / & + layerThicknessNew(k,iCell) + + activeTracerVerticalAdvectionTendency(iTracer,k,iCell) = & + activeTracerVerticalAdvectionTendency(iTracer,k,iCell) / & + layerThicknessNew(k,iCell) + + activeTracerHorMixTendency(iTracer,k,iCell) = & + activeTracerHorMixTendency(iTracer,k,iCell) / & + layerThicknessNew(k,iCell) + + activeTracerSurfaceFluxTendency(iTracer,k,iCell) = & + activeTracerSurfaceFluxTendency(iTracer,k,iCell) / & + layerThicknessNew(k,iCell) + + activeTracerNonLocalTendency(iTracer,k,iCell) = & + activeTracerNonLocalTendency(iTracer,k,iCell) / & + layerThicknessNew(k,iCell) + end do +#else activeTracerHorizontalAdvectionTendency(:,k,iCell) = & activeTracerHorizontalAdvectionTendency(:,k,iCell) / & layerThicknessNew(k,iCell) @@ -1834,13 +1918,14 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ activeTracerSurfaceFluxTendency(:,k,iCell) / & layerThicknessNew(k,iCell) - temperatureShortWaveTendency(k,iCell) = & - temperatureShortWaveTendency(k,iCell) / & - layerThicknessNew(k,iCell) - activeTracerNonLocalTendency(:,k,iCell) = & activeTracerNonLocalTendency(:,k,iCell) / & layerThicknessNew(k,iCell) +#endif + + temperatureShortWaveTendency(k,iCell) = & + temperatureShortWaveTendency(k,iCell) / & + layerThicknessNew(k,iCell) end do end do #ifndef MPAS_OPENACC @@ -1849,6 +1934,18 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ #endif endif +#ifdef MPAS_OPENACC + !$acc end parallel +#endif + +! This tracer block is still computed on the CPU +#ifdef MPAS_OPENACC + !$acc update host(layerThicknessNew) + !$acc update host(layerThicknessCur, layerThicknessTend) + !$acc update host(activeTracersNew) + !$acc update host(tracersSurfaceValue) +#endif + call mpas_pool_begin_iteration(tracersPool) do while (mpas_pool_get_next_member(tracersPool, & groupItr) ) @@ -1963,10 +2060,29 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ end if ! tracer end do ! tracer group +#ifdef MPAS_OPENACC + !$acc update device(layerThicknessNew) + !$acc update device(activeTracersNew) +#endif + if (config_use_freq_filtered_thickness) then +#ifdef MPAS_OPENACC + !$acc parallel present(minLevelCell, maxLevelCell) & + !$acc present(lowFreqDivergenceNew) & + !$acc present(lowFreqDivergenceCur) & + !$acc present(lowFreqDivergenceTend) +#endif + +#ifdef MPAS_OPENACC + !$acc loop gang +#else !$omp parallel !$omp do schedule(runtime) private(k) +#endif do iCell = 1, nCellsAll +#ifdef MPAS_OPENACC + !$acc loop vector +#endif do k = minLevelCell(iCell), maxLevelCell(iCell) ! h^{hf}_{n+1} was computed in Stage 1 @@ -1982,6 +2098,10 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ !$omp end parallel #endif +#ifdef MPAS_OPENACC + !$acc end parallel +#endif + end if ! Recompute final u to go on to next step. @@ -2000,9 +2120,24 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ ! so normalBaroclinicVelocity does not have to be ! recomputed here. +#ifdef MPAS_OPENACC + !$acc parallel present(minLevelEdgeBot, maxLevelEdgeTop) & + !$acc present(normalVelocityNew) & + !$acc present(normalBarotropicVelocityNew) & + !$acc present(normalBaroclinicVelocityNew) & + !$acc present(normalBaroclinicVelocityCur) +#endif + +#ifdef MPAS_OPENACC + !$acc loop gang +#else !$omp parallel !$omp do schedule(runtime) private(k) +#endif do iEdge = 1, nEdgesAll +#ifdef MPAS_OPENACC + !$acc loop vector +#endif do k = minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) normalVelocityNew(k,iEdge) = & normalBarotropicVelocityNew(iEdge) + & @@ -2015,6 +2150,34 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ !$omp end parallel #endif +#ifdef MPAS_OPENACC + !$acc end parallel +#endif + +#ifdef MPAS_OPENACC + !$acc exit data copyout(layerThicknessNew) + !$acc exit data copyout(activeTracersNew) + !$acc exit data delete(layerThicknessTend, layerThicknessCur) + !$acc exit data copyout(normalVelocityNew) + !$acc exit data delete(normalBarotropicVelocityNew) + !$acc exit data delete(normalBaroclinicVelocityNew) + !$acc exit data delete(normalBaroclinicVelocityCur) + if (varinp_compute_active_tracer_budgets) then + !$acc exit data copyout(activeTracerHorizontalAdvectionTendency) + !$acc exit data copyout(activeTracerHorMixTendency) + !$acc exit data copyout(activeTracerVerticalAdvectionTendency) + !$acc exit data copyout(activeTracerSurfaceFluxTendency) + !$acc exit data copyout(activeTracerNonLocalTendency) + !$acc exit data copyout(temperatureShortWaveTendency) + !$acc exit data delete(layerThickEdgeFlux) + !$acc exit data copyout(activetracerhorizontaladvectionedgeflux) + endif + if (config_use_freq_filtered_thickness) then + !$acc exit data copyout(lowFreqDivergenceNew) + !$acc exit data delete(lowFreqDivergenceCur, lowFreqDivergenceTend) + endif +#endif + endif ! splitExplicitStep call mpas_timer_stop('se loop fini') From 2415a02720e765535bbfdd4d609ae22d8dc15274 Mon Sep 17 00:00:00 2001 From: Bob Robey Date: Sat, 14 May 2022 13:13:05 -0600 Subject: [PATCH 2/6] Moving data managementment outside of explicitTimeStep block --- .../mpas_ocn_time_integration_split.F | 214 +++++++++--------- 1 file changed, 111 insertions(+), 103 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index e28e63fb8de0..0b1a1f50511e 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -1569,6 +1569,50 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ meshPool, swForcingPool, & dt, activeTracersOnly, 2) +#ifdef MPAS_OPENACC + !$acc enter data copyin(layerThicknessNew, normalVelocityNew) + !$acc enter data copyin(normalBarotropicVelocityNew, & + !$acc normalBaroclinicVelocityNew) + !$acc enter data copyin(activeTracersNew) + !$acc update device(layerThickEdgeFlux) + if (config_use_freq_filtered_thickness) then + !$acc enter data copyin(lowFreqDivergenceNew) + !$acc enter data copyin(lowFreqDivergenceCur) + !$acc enter data copyin(lowFreqDivergenceTend) + endif + if (splitExplicitStep < numTSIterations) then + !$acc update device (normalTransportVelocity, & + !$acc normalGMBolusVelocity) + !$acc enter data copyin(atmosphericPressure, seaIcePressure) + !$acc enter data copyin(sshNew) + !$acc update device(tracersSurfaceValue) + if ( associated(frazilSurfacePressure) ) then + !$acc enter data copyin(frazilSurfacePressure) + endif + if (landIcePressureOn) then + !$acc enter data copyin(landIcePressure) + !$acc enter data copyin(landIceDraft) + endif + if (config_use_freq_filtered_thickness) then + !$acc enter data copyin(highFreqThicknessNew) + !$acc enter data copyin(highFreqThicknessCur) + endif + elseif (splitExplicitStep == numTSIterations) then + !$acc enter data copyin(layerThicknessCur) + !$acc enter data copyin(layerThicknessTend) + !$acc enter data copyin(normalBaroclinicVelocityCur) + if (varinp_compute_active_tracer_budgets) then + !$acc update device(activeTracerHorizontalAdvectionEdgeFlux) + !$acc update device(activeTracerHorizontalAdvectionTendency) + !$acc update device(activeTracerVerticalAdvectionTendency) + !$acc update device(activeTracerHorMixTendency) + !$acc update device(activeTracerSurfaceFluxTendency) + !$acc update device(temperatureShortWaveTendency) + !$acc update device(activeTracerNonLocalTendency) + endif + endif +#endif + call mpas_timer_stop('se tracer tend') ! update halo for tracer tendencies @@ -1614,8 +1658,15 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ ! Only need T & S for earlier iterations, ! then all the tracers needed the last time through. +!! Keeping this calc on the CPU for now +#ifdef MPAS_OPENACC + !$acc update host(layerThicknessNew) + !$acc update host(activeTracersNew) + !$acc update host(tracersSurfaceValue) +#else !$omp parallel !$omp do schedule(runtime) private(i, k, temp_h, temp) +#endif do iCell = 1, nCellsAll do k = minLevelCell(iCell), maxLevelCell(iCell) @@ -1639,33 +1690,14 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ end do ! tracer index end do ! vertical end do ! iCell +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif #ifdef MPAS_OPENACC - !$acc enter data copyin(layerThicknessNew, normalVelocityNew) - !$acc update device (normalTransportVelocity, & - !$acc normalGMBolusVelocity) - !$acc enter data copyin(atmosphericPressure, seaIcePressure) - !$acc enter data copyin(sshNew) - !$acc enter data copyin(activeTracersNew) - !$acc update device(tracersSurfaceValue) - if ( associated(frazilSurfacePressure) ) then - !$acc enter data copyin(frazilSurfacePressure) - endif - if (landIcePressureOn) then - !$acc enter data copyin(landIcePressure) - !$acc enter data copyin(landIceDraft) - endif - !$acc enter data copyin(normalBarotropicVelocityNew, & - !$acc normalBaroclinicVelocityNew) - if (config_use_freq_filtered_thickness) then - !$acc enter data copyin(highFreqThicknessNew) - !$acc enter data copyin(highFreqThicknessCur) - !$acc enter data copyin(lowFreqDivergenceNew) - !$acc enter data copyin(lowFreqDivergenceCur) - !$acc enter data copyin(lowFreqDivergenceTend) - endif + !$acc update device(layerThicknessNew) + !$acc update device(activeTracersNew) #endif if (config_use_freq_filtered_thickness) then @@ -1737,48 +1769,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ call ocn_diagnostic_solve(dt, statePool, forcingPool, & meshPool, scratchPool, & tracersPool, 2, full=.false.) -#ifdef MPAS_OPENACC - !$acc update host(layerThickEdgeFlux, layerThickEdgeMean) - !$acc update host(relativeVorticity, circulation) - !$acc update host(vertTransportVelocityTop, & - !$acc vertGMBolusVelocityTop, & - !$acc relativeVorticityCell, & - !$acc divergence, & - !$acc kineticEnergyCell, & - !$acc tangentialVelocity, & - !$acc vertVelocityTop) - !$acc update host(normRelVortEdge, normPlanetVortEdge, & - !$acc normalizedRelativeVorticityCell) - !$acc update host (surfacePressure) - !$acc update host(zMid, zTop) - !$acc exit data copyout(sshNew) - !$acc exit data delete(activeTracersNew) - !$acc update host(tracersSurfaceValue) - !$acc update host(normalVelocitySurfaceLayer) - !$acc exit data delete (atmosphericPressure, seaIcePressure) - if ( associated(frazilSurfacePressure) ) then - !$acc exit data delete(frazilSurfacePressure) - endif - if (landIcePressureOn) then - !$acc exit data delete(landIcePressure) - !$acc exit data delete(landIceDraft) - endif - !$acc exit data delete(layerThicknessNew) - !$acc exit data delete(normalBarotropicVelocityNew, & - !$acc normalBaroclinicVelocityNew) - !$acc exit data copyout(normalVelocityNew) - !$acc update host(density, potentialDensity, displacedDensity) - !$acc update host(thermExpCoeff, & - !$acc& salineContractCoeff) - !$acc update host(montgomeryPotential, pressure) - if (config_use_freq_filtered_thickness) then - !$acc exit data copyout(highFreqThicknessNew) - !$acc exit data delete(highFreqThicknessCur) - !$acc exit data copyout(lowFreqDivergenceNew) - !$acc exit data delete(lowFreqDivergenceCur) - !$acc exit data delete(lowFreqDivergenceTend) - endif -#endif !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! If large iteration complete, compute all variables at @@ -1787,24 +1777,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ elseif (splitExplicitStep == numTSIterations) then -#ifdef MPAS_OPENACC - !$acc enter data copyin(layerThicknessTend, layerThicknessNew, layerThicknessCur) - !$acc enter data copyin(activeTracersNew) - !$acc enter data copyin(normalVelocityNew) - !$acc enter data copyin(normalBarotropicVelocityNew) - !$acc enter data copyin(normalBaroclinicVelocityNew) - !$acc enter data copyin(normalBaroclinicVelocityCur) - if (varinp_compute_active_tracer_budgets) then - !$acc enter data copyin(layerThickEdgeFlux, activetracerhorizontaladvectionedgeflux) - !$acc enter data copyin(activeTracerHorizontalAdvectionTendency, activeTracerHorMixTendency, activeTracerVerticalAdvectionTendency, & - !$acc activeTracerSurfaceFluxTendency, activeTracerNonLocalTendency, temperatureShortWaveTendency) - endif - if (config_use_freq_filtered_thickness) then - !$acc enter data copyin(lowFreqDivergenceNew, lowFreqDivergenceCur, lowFreqDivergenceTend) - endif -#endif - - #ifdef MPAS_OPENACC !$acc parallel present(minLevelCell, maxLevelCell) & !$acc present(layerThicknessTend) & @@ -2154,32 +2126,68 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ !$acc end parallel #endif + endif ! splitExplicitStep + #ifdef MPAS_OPENACC - !$acc exit data copyout(layerThicknessNew) - !$acc exit data copyout(activeTracersNew) - !$acc exit data delete(layerThicknessTend, layerThicknessCur) - !$acc exit data copyout(normalVelocityNew) - !$acc exit data delete(normalBarotropicVelocityNew) - !$acc exit data delete(normalBaroclinicVelocityNew) - !$acc exit data delete(normalBaroclinicVelocityCur) - if (varinp_compute_active_tracer_budgets) then - !$acc exit data copyout(activeTracerHorizontalAdvectionTendency) - !$acc exit data copyout(activeTracerHorMixTendency) - !$acc exit data copyout(activeTracerVerticalAdvectionTendency) - !$acc exit data copyout(activeTracerSurfaceFluxTendency) - !$acc exit data copyout(activeTracerNonLocalTendency) - !$acc exit data copyout(temperatureShortWaveTendency) - !$acc exit data delete(layerThickEdgeFlux) - !$acc exit data copyout(activetracerhorizontaladvectionedgeflux) + !$acc exit data copyout(layerThicknessNew) + !$acc exit data copyout(normalVelocityNew) + !$acc exit data delete(normalBarotropicVelocityNew, & + !$acc normalBaroclinicVelocityNew) + !$acc exit data copyout(activeTracersNew) + !$acc update host(layerThickEdgeFlux) + if (config_use_freq_filtered_thickness) then + !$acc exit data copyout(lowFreqDivergenceNew) + !$acc exit data delete(lowFreqDivergenceCur) + !$acc exit data delete(lowFreqDivergenceTend) + endif + if (splitExplicitStep < numTSIterations) then + !$acc exit data delete (atmosphericPressure, seaIcePressure) + !$acc exit data copyout(sshNew) + !$acc update host(layerThickEdgeMean) + !$acc update host(relativeVorticity, circulation) + !$acc update host(vertTransportVelocityTop, & + !$acc vertGMBolusVelocityTop, & + !$acc relativeVorticityCell, & + !$acc divergence, & + !$acc kineticEnergyCell, & + !$acc tangentialVelocity, & + !$acc vertVelocityTop) + !$acc update host(normRelVortEdge, normPlanetVortEdge, & + !$acc normalizedRelativeVorticityCell) + !$acc update host (surfacePressure) + !$acc update host(zMid, zTop) + !$acc update host(tracersSurfaceValue) + !$acc update host(normalVelocitySurfaceLayer) + !$acc update host(density, potentialDensity, displacedDensity) + !$acc update host(thermExpCoeff, & + !$acc& salineContractCoeff) + !$acc update host(montgomeryPotential, pressure) + if (landIcePressureOn) then + !$acc exit data delete(landIcePressure) + !$acc exit data delete(landIceDraft) endif if (config_use_freq_filtered_thickness) then - !$acc exit data copyout(lowFreqDivergenceNew) - !$acc exit data delete(lowFreqDivergenceCur, lowFreqDivergenceTend) + !$acc exit data copyout(highFreqThicknessNew) + !$acc exit data delete(highFreqThicknessCur) endif + if ( associated(frazilSurfacePressure) ) then + !$acc exit data delete(frazilSurfacePressure) + endif + elseif (splitExplicitStep == numTSIterations) then + !$acc exit data delete(layerThicknessCur, layerThicknessTend) + !$acc exit data delete(normalBaroclinicVelocityCur) + if (varinp_compute_active_tracer_budgets) then + !$acc update host(activeTracerHorizontalAdvectionEdgeFlux) + !$acc update host(activeTracerHorizontalAdvectionTendency) + !$acc update host(activeTracerVerticalAdvectionTendency) + !$acc update host(activeTracerHorMixTendency) + !$acc update host(activeTracerSurfaceFluxTendency) + !$acc update host(temperatureShortWaveTendency) + !$acc update host(activeTracerNonLocalTendency) + endif + endif #endif - endif ! splitExplicitStep - call mpas_timer_stop('se loop fini') call mpas_timer_stop('se loop') From 1f2f213d968087a4de5d56fe2d9b9c0e6642337c Mon Sep 17 00:00:00 2001 From: Bob Robey Date: Sat, 14 May 2022 15:27:33 -0600 Subject: [PATCH 3/6] Fine-tuning extendsplit region --- .../mpas_ocn_time_integration_split.F | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index 0b1a1f50511e..de4b277c194e 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -1569,6 +1569,15 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ meshPool, swForcingPool, & dt, activeTracersOnly, 2) + call mpas_pool_get_array(tracersPool, 'activeTracers', & + tracersGroupCur, 1) + call mpas_pool_get_array(tracersPool, 'activeTracers', & + tracersGroupNew, 2) + call mpas_pool_get_array(statePool, 'normalVelocity', & + normalVelocityCur, 1) + call mpas_pool_get_array(tracersTendPool,'activeTracersTend', & + activeTracersTend) + #ifdef MPAS_OPENACC !$acc enter data copyin(layerThicknessNew, normalVelocityNew) !$acc enter data copyin(normalBarotropicVelocityNew, & @@ -1634,14 +1643,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ call mpas_timer_stop("se halo tracers") call mpas_timer_start('se loop fini') - call mpas_pool_get_array(tracersPool, 'activeTracers', & - tracersGroupCur, 1) - call mpas_pool_get_array(tracersPool, 'activeTracers', & - tracersGroupNew, 2) - call mpas_pool_get_array(statePool, 'normalVelocity', & - normalVelocityCur, 1) - call mpas_pool_get_array(tracersTendPool,'activeTracersTend', & - activeTracersTend) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! If iterating, reset variables for next iteration @@ -2224,6 +2225,9 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ #endif call ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, & scratchPool, tracersPool, 2) + + call mpas_dmpar_field_halo_exch(domain, 'surfaceFrictionVelocity') + #ifdef MPAS_OPENACC !$acc update host(layerThickEdgeFlux, layerThickEdgeMean) !$acc update host(relativeVorticity, circulation) @@ -2264,8 +2268,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ !$acc exit data delete(layerThicknessNew, normalVelocityNew) #endif - call mpas_dmpar_field_halo_exch(domain, 'surfaceFrictionVelocity') - ! Compute normalGMBolusVelocity; it will be added to the ! baroclinic modes in Stage 2 above. !if (config_use_GM.or.config_use_Redi) then From 47e44ff767d0609b8d22f041b2263c485ef04b50 Mon Sep 17 00:00:00 2001 From: Bob Robey Date: Mon, 16 May 2022 19:25:09 -0600 Subject: [PATCH 4/6] Fixing remaining loop in diagnostic_solve to run on GPU --- .../mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index d43e2bb7a8f6..abb537b5a029 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -110,7 +110,7 @@ subroutine ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, scratchPoo integer, intent(in), optional :: timeLevelIn !< Input: Time level in state logical, intent(in), optional :: full !< Input: Run all computations? - integer :: iEdge, iCell + integer :: iEdge, iCell, iTracer integer :: err, nCells, nEdges, nVertices real (kind=RKIND) :: coef_3rd_order @@ -331,16 +331,21 @@ subroutine ocn_diagnostic_solve(dt, statePool, forcingPool, meshPool, scratchPoo ! tracersSurfaceValue calc ! inputs: activeTracers, minLevelCell ! outputs: tracersSurfaceValue -! !$acc parallel loop !!!present(tracersSurfaceValue, activeTracers, minLevelCell) + !$acc kernels present(tracersSurfaceValue, activeTracers, minLevelCell) #else !$omp parallel !$omp do schedule(runtime) +#endif +#ifdef MPAS_OPENACC + !$acc loop independent #endif do iCell = 1, nCells - tracersSurfaceValue(:, iCell) = activeTracers(:, minLevelCell(iCell), iCell) + do iTracer = 1,size(tracersSurfaceValue,1) + tracersSurfaceValue(iTracer, iCell) = activeTracers(iTracer, minLevelCell(iCell), iCell) + enddo end do #ifdef MPAS_OPENACC - !$acc update device(tracersSurfaceValue) + !$acc end kernels #else !$omp end do #endif From 1cd7009c2810a6d99c01ee35523e8abf8b04f2d8 Mon Sep 17 00:00:00 2001 From: Bob Robey Date: Thu, 26 May 2022 09:58:16 -0600 Subject: [PATCH 5/6] fixing another config variable --- .../src/mode_forward/mpas_ocn_time_integration_split.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index de4b277c194e..9b9adfe62faa 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -1817,7 +1817,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ !$omp end parallel #endif - if (varinp_compute_active_tracer_budgets) then + if (config_compute_active_tracer_budgets) then #ifdef MPAS_OPENACC !$acc loop gang From d2f2da00ab62459522f02da47a6b3cee53959867 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Thu, 26 May 2022 14:46:38 -0600 Subject: [PATCH 6/6] change two more varinp variables --- .../src/mode_forward/mpas_ocn_time_integration_split.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index 9b9adfe62faa..a474bb1f2305 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -1610,7 +1610,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ !$acc enter data copyin(layerThicknessCur) !$acc enter data copyin(layerThicknessTend) !$acc enter data copyin(normalBaroclinicVelocityCur) - if (varinp_compute_active_tracer_budgets) then + if (config_compute_active_tracer_budgets) then !$acc update device(activeTracerHorizontalAdvectionEdgeFlux) !$acc update device(activeTracerHorizontalAdvectionTendency) !$acc update device(activeTracerVerticalAdvectionTendency) @@ -2177,7 +2177,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ elseif (splitExplicitStep == numTSIterations) then !$acc exit data delete(layerThicknessCur, layerThicknessTend) !$acc exit data delete(normalBaroclinicVelocityCur) - if (varinp_compute_active_tracer_budgets) then + if (config_compute_active_tracer_budgets) then !$acc update host(activeTracerHorizontalAdvectionEdgeFlux) !$acc update host(activeTracerHorizontalAdvectionTendency) !$acc update host(activeTracerVerticalAdvectionTendency)