Output time averaged CASA fluxes and pools

Currently the MPI implementation does not output time-averaged pools and
fluxes (#566). This change implements the time-averaging functionality
which exists in the serial driver into the MPI master driver. This is
required to restore bitwise reproducibility in the CASA netcdf output
file across serial and MPI runs.

Author: SeanBryan51

src/offline/cable_mpimaster.F90

@@ -248,7 +248,9 @@ SUBROUTINE mpidrv_master (comm, trunk_sumbal, dels, koffset, kend, PLUME, CRU)
          ctime = 0,  &  ! day count for casacnp
          YYYY,       &  !
          LOY,        &  ! Length of Year
-         maxdiff(2)     ! location of maximum in convergence test
+         maxdiff(2), &  ! location of maximum in convergence test
+         count_sum_casa ! number of time steps over which casa pools &
+                                !and fluxes are aggregated (for output)
 
     CHARACTER :: dum*9, str1*9, str2*9, str3*9  ! dummy char for fileName generation
 
@@ -602,8 +604,8 @@ SUBROUTINE mpidrv_master (comm, trunk_sumbal, dels, koffset, kend, PLUME, CRU)
             CALL master_restart_types (comm, canopy, air, bgc)
           END IF
 
-          !  CALL zero_sum_casa(sum_casapool, sum_casaflux)
-          !  count_sum_casa = 0
+          CALL zero_sum_casa(sum_casapool, sum_casaflux)
+          count_sum_casa = 0
 
           ! CALL master_sumcasa_types(comm, sum_casapool, sum_casaflux)
           IF( icycle>0 .AND. spincasa) THEN
@@ -766,6 +768,13 @@ SUBROUTINE mpidrv_master (comm, trunk_sumbal, dels, koffset, kend, PLUME, CRU)
               ! receive casa update from worker
               CALL master_receive (ocomm, oktau, casa_ts)
 
+              IF(MOD((oktau-kstart+1),ktauday)==0) THEN
+                ! update time-aggregates of casa pools and fluxes
+                CALL update_sum_casa(sum_casapool, sum_casaflux, casapool, casaflux, &
+                    & .TRUE. , .FALSE., 1)
+                count_sum_casa = count_sum_casa + 1
+              END IF
+
               CALL MPI_Waitall (wnp, recv_req, recv_stats, ierr)
               ! receive casa dump requirements from worker
               IF ( ((.NOT.spinup).OR.(spinup.AND.spinConv)) .AND.   &
@@ -835,11 +844,12 @@ SUBROUTINE mpidrv_master (comm, trunk_sumbal, dels, koffset, kend, PLUME, CRU)
               IF ( IS_CASA_TIME("write", yyyy, oktau, kstart, &
                                 koffset, kend, ktauday, logn) ) THEN
                 ctime = ctime +1
-
-
-                CALL WRITE_CASA_OUTPUT_NC (veg, casamet, casapool, casabal, casaflux, &
-                     CASAONLY, ctime, &
-                     ( ktau.EQ.kend .AND. YYYY .EQ.cable_user%YearEnd ) )
+                CALL update_sum_casa(sum_casapool, sum_casaflux, casapool, casaflux, &
+                    .FALSE. , .TRUE. , count_sum_casa)
+                CALL WRITE_CASA_OUTPUT_NC (veg, casamet, sum_casapool, casabal, sum_casaflux, &
+                    CASAONLY, ctime, ( oktau == kend .AND. YYYY == cable_user%YearEnd ) )
+                count_sum_casa = 0
+                CALL zero_sum_casa(sum_casapool, sum_casaflux)
               ENDIF
             ENDIF
 
@@ -952,6 +962,13 @@ SUBROUTINE mpidrv_master (comm, trunk_sumbal, dels, koffset, kend, PLUME, CRU)
 
             CALL master_receive (ocomm, oktau, casa_ts)
 
+            IF(MOD((oktau-kstart+1),ktauday)==0) THEN
+              ! update time-aggregates of casa pools and fluxes
+              CALL update_sum_casa(sum_casapool, sum_casaflux, casapool, casaflux, &
+                  & .TRUE. , .FALSE., 1)
+              count_sum_casa = count_sum_casa + 1
+            END IF
+
             IF ( ((.NOT.spinup).OR.(spinup.AND.spinConv)) .AND. &
                  ( IS_CASA_TIME("dwrit", yyyy, oktau, kstart, &
                  koffset, kend, ktauday, logn) ) ) THEN
@@ -1024,9 +1041,12 @@ SUBROUTINE mpidrv_master (comm, trunk_sumbal, dels, koffset, kend, PLUME, CRU)
         IF((.NOT.spinup).OR.(spinup.AND.spinConv)) THEN
           IF(icycle >0) THEN
             ctime = ctime +1
-            CALL WRITE_CASA_OUTPUT_NC (veg, casamet, casapool, casabal, casaflux, &
-                 CASAONLY, ctime, ( ktau.EQ.kend .AND. YYYY .EQ.               &
-                 cable_user%YearEnd ) )
+            CALL update_sum_casa(sum_casapool, sum_casaflux, casapool, casaflux, &
+                .FALSE. , .TRUE. , count_sum_casa)
+            CALL WRITE_CASA_OUTPUT_NC (veg, casamet, sum_casapool, casabal, sum_casaflux, &
+                CASAONLY, ctime, ( oktau == kend .AND. YYYY == cable_user%YearEnd ) )
+            count_sum_casa = 0
+            CALL zero_sum_casa(sum_casapool, sum_casaflux)
             IF ( cable_user%CALL_POP ) THEN
 
               ! CALL master_receive_pop(POP, ocomm)