Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

parallel netcdf IO #41

Merged
merged 33 commits into from
Feb 4, 2020
Merged

parallel netcdf IO #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
45d0ce2
netcdf parallel writing and lon/lat in netcdf file
junwang-noaa Jan 8, 2020
6b9b865
some changes to get it to (almost) run on hera
jswhit2 Jan 8, 2020
ccdbf27
lat should be in degrees - remove conversion to radians.
jswhit2 Jan 9, 2020
b3e52a0
import updates from jswhit/fv3atm
jswhit2 Jan 9, 2020
a201d6c
more bug fixes - now works on hera
jswhit2 Jan 9, 2020
656c546
specify collective access if compression turned on.
jswhit2 Jan 9, 2020
ade86ed
use classic model
jswhit2 Jan 9, 2020
49ffb5e
bug fixes for parallel IO with compression
jswhit2 Jan 10, 2020
134ebc0
fix calculation of max compression error
jswhit2 Jan 10, 2020
5fce869
turn off shuffle filter
jswhit2 Jan 10, 2020
30516fc
code simplification
jswhit2 Jan 10, 2020
930d622
remove debug print
jswhit2 Jan 17, 2020
6cb687d
don't use parallel IO for 2d file (since it seems to increase run time)
jswhit2 Jan 22, 2020
c883717
allow multiple values of output_file, as long as they all start with …
jswhit2 Jan 23, 2020
ffb33e0
use default chunksize for 2d vars
jswhit2 Jan 24, 2020
89184bd
delete commented out macro ESMF_ERR_ABORT
jswhit2 Jan 24, 2020
5af9ccf
delete rad2dg
jswhit2 Jan 24, 2020
b49ef58
add module_write_netcdf_parallel.F90
jswhit2 Jan 25, 2020
2e63f49
add option to build without parallel netcdf (-DNO_PARALLEL_NETCDF)
jswhit2 Jan 25, 2020
4a45855
fix typo
jswhit2 Jan 25, 2020
14dfcab
stub file for building without parallel netcdf lib
jswhit2 Jan 26, 2020
5beba44
Merge remote-tracking branch 'upstream/develop' into netcdf_parallel
jswhit2 Jan 26, 2020
dab1dd8
allow chunksizes for 2d arrays to be set in model_configure (ichunk2d…
jswhit2 Jan 26, 2020
a6c36fb
add ichunk3d,jchunk3d,kchunk3d to specify 3d chunksizes.
jswhit2 Jan 27, 2020
1cf5a6c
fix typo
jswhit2 Jan 27, 2020
a3f0306
put ifdefs in module_write_netcdf_parallel.F90 so no stub file needed
jswhit2 Jan 27, 2020
1525cc0
don't need this file anymore
jswhit2 Jan 27, 2020
51d3dc1
remove module_write_netcdf_parallel_stub.o target
jswhit2 Jan 27, 2020
8ddde89
use specified chunksizes for serial IO. If chunksize parameter negati…
jswhit2 Jan 27, 2020
98948bd
update comments
jswhit2 Jan 27, 2020
45d35d5
get output_file without esmf call error
junwang-noaa Jan 30, 2020
a0f9ffa
syntax fix
junwang-noaa Jan 30, 2020
715f1ae
fix stub interface in module_write_netcdf_parallel.F90 for cmake build
junwang-noaa Feb 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 39 additions & 6 deletions fv3_cap.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ module fv3gfs_cap_mod
wrttasks_per_group, n_group, &
lead_wrttask, last_wrttask, &
output_grid, output_file, &
imo, jmo, write_nemsioflip, &
imo,jmo,ichunk2d,jchunk2d,write_nemsioflip,&
ichunk3d,jchunk3d,kchunk3d, &
write_fsyncflag, nsout_io, &
cen_lon, cen_lat, ideflate, &
lon1, lat1, lon2, lat2, dlon, dlat, &
Expand Down Expand Up @@ -235,8 +236,9 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)

integer,dimension(6) :: date, date_init
integer :: mpi_comm_atm
integer :: i, j, k, io_unit, urc
integer :: i, j, k, io_unit, urc, ierr
integer :: petcount, mype
integer :: num_output_file
logical :: isPetLocal
logical :: OPENED
character(ESMF_MAXSTR) :: name
Expand Down Expand Up @@ -307,6 +309,14 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
call ESMF_ConfigGetAttribute(config=CF,value=iau_offset,default=0,label ='iau_offset:',rc=rc)
if (iau_offset < 0) iau_offset=0

! chunksizes for netcdf_parallel
call ESMF_ConfigGetAttribute(config=CF,value=ichunk2d,default=0,label ='ichunk2d:',rc=rc)
call ESMF_ConfigGetAttribute(config=CF,value=jchunk2d,default=0,label ='jchunk2d:',rc=rc)
call ESMF_ConfigGetAttribute(config=CF,value=ichunk3d,default=0,label ='ichunk3d:',rc=rc)
call ESMF_ConfigGetAttribute(config=CF,value=jchunk3d,default=0,label ='jchunk3d:',rc=rc)
call ESMF_ConfigGetAttribute(config=CF,value=kchunk3d,default=0,label ='kchunk3d:',rc=rc)

! zlib compression flag
call ESMF_ConfigGetAttribute(config=CF,value=ideflate,default=0,label ='ideflate:',rc=rc)
if (ideflate < 0) ideflate=0

Expand Down Expand Up @@ -346,8 +356,33 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
CALL ESMF_ConfigGetAttribute(config=CF,value=filename_base(i), rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
enddo
if(mype == 0) print *,'af nems config,num_files=',num_files, &
'filename_base=',filename_base

allocate(output_file(num_files))
num_output_file = ESMF_ConfigGetLen(config=CF, label ='output_file:',rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
if (num_files == num_output_file) then
CALL ESMF_ConfigGetAttribute(CF,valueList=output_file,label='output_file:', &
count=num_files, rc=RC)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
do i = 1, num_files
if(output_file(i) /= "netcdf" .and. output_file(i) /= "netcdf_parallel") then
write(0,*)"fv3_cap.F90: only netcdf and netcdf_parallel are allowed for multiple values of output_file"
call ESMF_Finalize(endflag=ESMF_END_ABORT)
endif
enddo
else if ( num_output_file == 1) then
CALL ESMF_ConfigGetAttribute(CF,valuelist=output_file,label='output_file:', count=1, rc=RC)
output_file(1:num_files) = output_file(1)
else
output_file(1:num_files) = 'netcdf'
endif
if(mype == 0) then
print *,'af nems config,num_files=',num_files
do i=1,num_files
print *,'num_file=',i,'filename_base= ',trim(filename_base(i)),&
' output_file= ',trim(output_file(i))
enddo
endif
!
! variables for alarms
call ESMF_ConfigGetAttribute(config=CF, value=nfhout, label ='nfhout:', rc=rc)
Expand All @@ -359,10 +394,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)

! variables for I/O options
call ESMF_ConfigGetAttribute(config=CF, value=output_grid, label ='output_grid:',rc=rc)
call ESMF_ConfigGetAttribute(config=CF, value=output_file, label ='output_file:',rc=rc)
if (mype == 0) then
print *,'output_grid=',trim(output_grid)
print *,'output_file=',trim(output_file)
end if
write_nemsioflip =.false.
write_fsyncflag =.false.
Expand Down
5 changes: 5 additions & 0 deletions io/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,18 @@ else()
add_definitions(-DNO_INLINE_POST)
endif()

if(NOT PARALLEL_NETCDF)
add_definitions(-DNO_PARALLEL_NETCDF)
endif()

add_library(
io

ffsync.F90
FV3GFS_io.F90
module_write_nemsio.F90
module_write_netcdf.F90
module_write_netcdf_parallel.F90
module_fv3_io_def.F90
module_write_internal_state.F90
module_wrt_grid_comp.F90
Expand Down
5 changes: 4 additions & 1 deletion io/makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ SRCS_F90 = \
$(POST_SRC) \
./module_write_nemsio.F90 \
./module_write_netcdf.F90 \
./module_write_netcdf_parallel.F90 \
./module_fv3_io_def.F90 \
./module_write_internal_state.F90 \
./module_wrt_grid_comp.F90
Expand Down Expand Up @@ -69,7 +70,9 @@ post_nems_routines.o: post_nems_routines.F90
module_write_nemsio.o: module_write_nemsio.F90
$(FC) $(CPPDEFS) $(CPPFLAGS) $(FPPFLAGS) $(FFLAGS) $(OTHERFLAGS) $(OTHER_FFLAGS) $(ESMF_INC) $(NEMSIOINC) -c module_write_nemsio.F90
module_write_netcdf.o: module_write_netcdf.F90
$(FC) $(CPPDEFS) $(CPPFLAGS) $(FPPFLAGS) $(FFLAGS) $(OTHERFLAGS) $(OTHER_FFLAGS) $(ESMF_INC) $(NEMSIOINC) -c module_write_netcdf.F90
$(FC) $(CPPDEFS) $(CPPFLAGS) $(FPPFLAGS) $(FFLAGS) $(OTHERFLAGS) $(OTHER_FFLAGS) $(ESMF_INC) -c module_write_netcdf.F90
module_write_netcdf_parallel.o: module_write_netcdf_parallel.F90
$(FC) $(CPPDEFS) $(CPPFLAGS) $(FPPFLAGS) $(FFLAGS) $(OTHERFLAGS) $(OTHER_FFLAGS) $(ESMF_INC) -c module_write_netcdf_parallel.F90
module_write_internal_state.o: module_write_internal_state.F90
$(FC) $(CPPDEFS) $(CPPFLAGS) $(FPPFLAGS) $(FFLAGS) $(OTHERFLAGS) $(OTHER_FFLAGS) $(ESMF_INC) -c module_write_internal_state.F90
module_wrt_grid_comp.o: module_wrt_grid_comp.F90
Expand Down
3 changes: 2 additions & 1 deletion io/module_fv3_io_def.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@ module module_fv3_io_def
integer :: num_files
character(255) :: app_domain
character(255) :: output_grid
character(255) :: output_file
integer :: imo,jmo
integer :: ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d
integer :: nbdlphys
integer :: nsout_io, iau_offset, ideflate, nbits
real :: cen_lon, cen_lat, lon1, lat1, lon2, lat2, dlon, dlat
real :: stdlat1, stdlat2, dx, dy
character(255),dimension(:),allocatable :: filename_base
character(255),dimension(:),allocatable :: output_file
!
integer,dimension(:),allocatable :: lead_wrttask, last_wrttask
!
Expand Down
106 changes: 59 additions & 47 deletions io/module_write_netcdf.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
!#define ESMF_ERR_ABORT(rc) \
!if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; \
! if (ESMF_LogFoundError(rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT)

#define ESMF_ERR_RETURN(rc) if (ESMF_LogFoundError(rc, msg="Breaking out of subroutine", line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT)

#define NC_ERR_STOP(status) \
Expand All @@ -22,14 +18,15 @@ module module_write_netcdf
contains

!----------------------------------------------------------------------------------------
subroutine write_netcdf(fieldbundle, wrtfb, filename, mpi_comm, mype, im, jm, rc)
subroutine write_netcdf(fieldbundle, wrtfb, filename, mpi_comm, mype, im, jm, ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d, rc)
!
type(ESMF_FieldBundle), intent(in) :: fieldbundle
type(ESMF_FieldBundle), intent(in) :: wrtfb
character(*), intent(in) :: filename
integer, intent(in) :: mpi_comm
integer, intent(in) :: mype
integer, intent(in) :: im, jm
integer, intent(in) :: ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d
integer, optional,intent(out) :: rc
!
!** local vars
Expand Down Expand Up @@ -69,9 +66,7 @@ subroutine write_netcdf(fieldbundle, wrtfb, filename, mpi_comm, mype, im, jm, rc
integer :: im_dimid, jm_dimid, pfull_dimid, phalf_dimid, time_dimid
integer :: im_varid, jm_varid, lm_varid, time_varid, lon_varid, lat_varid
integer, dimension(:), allocatable :: varids
!
!!
!
logical shuffle

call ESMF_FieldBundleGet(fieldbundle, fieldCount=fieldCount, rc=rc); ESMF_ERR_RETURN(rc)

Expand Down Expand Up @@ -117,15 +112,10 @@ subroutine write_netcdf(fieldbundle, wrtfb, filename, mpi_comm, mype, im, jm, rc
! create netcdf file and enter define mode
if (mype==0) then

if (ideflate == 0) then
ncerr = nf90_create(trim(filename), cmode=IOR(IOR(NF90_CLOBBER,NF90_64BIT_OFFSET),NF90_SHARE), &
ncid=ncid); NC_ERR_STOP(ncerr)
ncerr = nf90_set_fill(ncid, NF90_NOFILL, oldMode); NC_ERR_STOP(ncerr)
else
ncerr = nf90_create(trim(filename), cmode=IOR(IOR(NF90_CLOBBER,NF90_NETCDF4),NF90_CLASSIC_MODEL), &
ncid=ncid); NC_ERR_STOP(ncerr)
ncerr = nf90_set_fill(ncid, NF90_NOFILL, oldMode); NC_ERR_STOP(ncerr)
endif
ncerr = nf90_create(trim(filename),&
cmode=IOR(IOR(NF90_CLOBBER,NF90_NETCDF4),NF90_CLASSIC_MODEL),&
ncid=ncid); NC_ERR_STOP(ncerr)
ncerr = nf90_set_fill(ncid, NF90_NOFILL, oldMode); NC_ERR_STOP(ncerr)

! define dimensions
ncerr = nf90_def_dim(ncid, "grid_xt", im, im_dimid); NC_ERR_STOP(ncerr)
Expand Down Expand Up @@ -158,32 +148,53 @@ subroutine write_netcdf(fieldbundle, wrtfb, filename, mpi_comm, mype, im, jm, rc
if (fldlev(i) == 1) then
if (typekind == ESMF_TYPEKIND_R4) then
if (ideflate > 0) then
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,time_dimid/), varids(i), &
shuffle=.true.,deflate_level=ideflate, &
chunksizes=(/im,jm,1/)); NC_ERR_STOP(ncerr)
if (ichunk2d < 0 .or. jchunk2d < 0) then
! let netcdf lib choose chunksize
! shuffle filter on for 2d fields (lossless compression)
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,time_dimid/), varids(i), &
shuffle=.true.,deflate_level=ideflate); NC_ERR_STOP(ncerr)
else
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,time_dimid/), varids(i), &
shuffle=.true.,deflate_level=ideflate,&
chunksizes=(/ichunk2d,jchunk2d,1/),cache_size=40*im*jm); NC_ERR_STOP(ncerr)
endif
else
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,time_dimid/), varids(i)); NC_ERR_STOP(ncerr)
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,time_dimid/), varids(i)); NC_ERR_STOP(ncerr)
endif
else if (typekind == ESMF_TYPEKIND_R8) then
ncerr = nf90_def_var(ncid, trim(fldName), NF90_DOUBLE, &
(/im_dimid,jm_dimid,time_dimid/), varids(i)); NC_ERR_STOP(ncerr)
ncerr = nf90_def_var(ncid, trim(fldName), NF90_DOUBLE, &
(/im_dimid,jm_dimid,time_dimid/), varids(i)); NC_ERR_STOP(ncerr)
else
write(0,*)'Unsupported typekind ', typekind
stop
write(0,*)'Unsupported typekind ', typekind
stop
end if
else if (fldlev(i) > 1) then
if (typekind == ESMF_TYPEKIND_R4) then
if (ideflate > 0) then
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,pfull_dimid,time_dimid/), varids(i), &
shuffle=.false.,deflate_level=ideflate, &
chunksizes=(/im,jm,1,1/)); NC_ERR_STOP(ncerr)
! shuffle filter off for 3d fields using lossy compression
if (nbits > 0) then
shuffle=.false.
else
shuffle=.true.
endif
if (ichunk3d < 0 .or. jchunk3d < 0 .or. kchunk3d < 0) then
! let netcdf lib choose chunksize
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,pfull_dimid,time_dimid/), varids(i), &
shuffle=shuffle,deflate_level=ideflate); NC_ERR_STOP(ncerr)
else
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,pfull_dimid,time_dimid/), varids(i), &
shuffle=shuffle,deflate_level=ideflate,&
chunksizes=(/ichunk3d,jchunk3d,kchunk3d,1/)); NC_ERR_STOP(ncerr)
endif
else
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,pfull_dimid,time_dimid/), varids(i)); NC_ERR_STOP(ncerr)
endif
(/im_dimid,jm_dimid,pfull_dimid,time_dimid/), varids(i)); NC_ERR_STOP(ncerr)
endif
else if (typekind == ESMF_TYPEKIND_R8) then
ncerr = nf90_def_var(ncid, trim(fldName), NF90_DOUBLE, &
(/im_dimid,jm_dimid,pfull_dimid,time_dimid/), varids(i)); NC_ERR_STOP(ncerr)
Expand Down Expand Up @@ -224,7 +235,7 @@ subroutine write_netcdf(fieldbundle, wrtfb, filename, mpi_comm, mype, im, jm, rc
call ESMF_AttributeGet(fcstField(i), convention="NetCDF", purpose="FV3", &
name=trim(attName), value=varr8val, &
rc=rc); ESMF_ERR_RETURN(rc)
if (trim(attName) /= '_FillValue' ) then
if (trim(attName) /= '_FillValue') then
! FIXME: _FillValue must be cast to var type for recent versions of netcdf
ncerr = nf90_put_att(ncid, varids(i), trim(attName), varr8val); NC_ERR_STOP(ncerr)
endif
Expand All @@ -241,23 +252,23 @@ subroutine write_netcdf(fieldbundle, wrtfb, filename, mpi_comm, mype, im, jm, rc

end do ! i=1,fieldCount

! write grid_xt, grid_yt attributes
! write grid_xt, grid_yt attributes
if (trim(output_grid) == 'gaussian_grid' .or. &
trim(output_grid) == 'regional_latlon') then
ncerr = nf90_put_att(ncid, im_varid, "long_name", "T-cell longitude"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, im_varid, "units", "degrees_E"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "long_name", "T-cell latiitude"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "units", "degrees_N"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, im_varid, "long_name", "T-cell longitude"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, im_varid, "units", "degrees_E"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "long_name", "T-cell latiitude"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "units", "degrees_N"); NC_ERR_STOP(ncerr)
else if (trim(output_grid) == 'rotated_latlon') then
ncerr = nf90_put_att(ncid, im_varid, "long_name", "rotated T-cell longiitude"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, im_varid, "units", "degrees"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "long_name", "rotated T-cell latiitude"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "units", "degrees"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, im_varid, "long_name", "rotated T-cell longiitude"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, im_varid, "units", "degrees"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "long_name", "rotated T-cell latiitude"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "units", "degrees"); NC_ERR_STOP(ncerr)
else if (trim(output_grid) == 'lambert_conformal') then
ncerr = nf90_put_att(ncid, im_varid, "long_name", "x-coordinate of projection"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, im_varid, "units", "meters"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "long_name", "y-coordinate of projection"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "units", "meters"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, im_varid, "long_name", "x-coordinate of projection"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, im_varid, "units", "meters"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "long_name", "y-coordinate of projection"); NC_ERR_STOP(ncerr)
ncerr = nf90_put_att(ncid, jm_varid, "units", "meters"); NC_ERR_STOP(ncerr)
endif

ncerr = nf90_enddef(ncid); NC_ERR_STOP(ncerr)
Expand Down Expand Up @@ -377,6 +388,7 @@ subroutine write_netcdf(fieldbundle, wrtfb, filename, mpi_comm, mype, im, jm, rc

deallocate(fcstField)
deallocate(varids)
deallocate(compress_err)

if (mype==0) then
ncerr = nf90_close(ncid=ncid); NC_ERR_STOP(ncerr)
Expand Down
Loading