Skip to content

Commit

Permalink
Merge pull request Unidata#1838 from DennisHeimbigner/emptyfill.dmh
Browse files Browse the repository at this point in the history 
Fix error where not converting fill data
  • Loading branch information
@WardF
WardF authored Sep 14, 2020
2 parents b5dd57c + 33a59e5 commit ff9d1b2
Show file tree
Hide file tree
Showing 14 changed files with 359 additions and 107 deletions.
25 changes: 22 additions & 3 deletions debug/cf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,20 @@ DB=1
#NOTUIL=1
FAST=1
#PROF=1
#BENCH=1

NCZARR=1
HDF5=1
#NCZARR=1
#HDF5=1
DAP=1
S3=1
#S3=1
#S3TEST=1
#SZIP=1
#HDF4=1
#PNETCDF=1
#PAR4=1

AWSSDK_DIR="/cygdrive/c/tools/aws-cpp-sdk-all"

#TESTSERVERS="149.165.169.123:8080,remotetest.unidata.ucar.edu"
#TESTSERVERS="localhost:8080,149.165.169.123:8080"
#TESTSERVERS="149.165.169.123:8080"
Expand Down Expand Up @@ -120,11 +123,27 @@ if test "x$NCZARR" != "x" ; then
FLAGS="$FLAGS --enable-nczarr"
fi

if test "x$S3" != x ; then
FLAGS="$FLAGS --enable-s3-sdk"
else
FLAGS="$FLAGS --disable-s3-sdk"
fi
if test "x$S3TEST" != x ; then
FLAGS="$FLAGS --enable-s3-tests"
export AWSSDK_DIR="${AWSSDK_DIR}"
LDFLAGS="$LDFLAGS $AWSSDK_DIR/bin/aws-cpp-sdk-s3.dll"
CPPFLAGS="$CPPFLAGS -I${AWSSDK_DIR}/include"
fi

if test "x${PROF}" = x1 ; then
CFLAGS="${CFLAGS} -pg"
LDFLAGS="${LDFLAGS} -pg"
fi

if test "x${BENCH}" = x1 ; then
FLAGS="$FLAGS --enable-benchmarks"
fi

export PATH
export CC
export CPPFLAGS
Expand Down
102 changes: 51 additions & 51 deletions libhdf5/hdf5var.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1894,6 +1894,38 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
log_dim_info(var, fdims, fmaxdims, start, count);
#endif

/* Check the type_info fields. */
assert(var->type_info && var->type_info->size &&
var->type_info->format_type_info);

/* Later on, we will need to know the size of this type in the
* file. */
file_type_size = var->type_info->size;

/* Are we going to convert any data? (No converting of compound or
* opaque types.) */
if (mem_nc_type != var->type_info->hdr.id &&
mem_nc_type != NC_COMPOUND && mem_nc_type != NC_OPAQUE)
{
/* We must convert - allocate a buffer. */
need_to_convert++;
if (var->ndims)
for (d2 = 0; d2 < var->ndims; d2++)
len *= countp[d2];
LOG((4, "converting data for var %s type=%d len=%d", var->hdr.name,
var->type_info->hdr.id, len));

/* If we're reading, we need bufr to have enough memory to store
* the data in the file. If we're writing, we need bufr to be
* big enough to hold all the data in the file's type. */
if (len > 0)
if (!(bufr = malloc(len * file_type_size)))
BAIL(NC_ENOMEM);
}
else
if (!bufr)
bufr = data;

/* Check dimension bounds. Remember that unlimited dimensions can
* get data beyond the length of the dataset, but within the
* lengths of the unlimited dimension(s). */
Expand Down Expand Up @@ -1957,14 +1989,6 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
}
}

/* Check the type_info fields. */
assert(var->type_info && var->type_info->size &&
var->type_info->format_type_info);

/* Later on, we will need to know the size of this type in the
* file. */
file_type_size = var->type_info->size;

if (!no_read)
{
/* Now you would think that no one would be crazy enough to write
Expand Down Expand Up @@ -2007,30 +2031,6 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
bufr = *(char **)data;
}

/* Are we going to convert any data? (No converting of compound or
* opaque types.) */
if (mem_nc_type != var->type_info->hdr.id &&
mem_nc_type != NC_COMPOUND && mem_nc_type != NC_OPAQUE)
{
/* We must convert - allocate a buffer. */
need_to_convert++;
if (var->ndims)
for (d2 = 0; d2 < var->ndims; d2++)
len *= countp[d2];
LOG((4, "converting data for var %s type=%d len=%d", var->hdr.name,
var->type_info->hdr.id, len));

/* If we're reading, we need bufr to have enough memory to store
* the data in the file. If we're writing, we need bufr to be
* big enough to hold all the data in the file's type. */
if (len > 0)
if (!(bufr = malloc(len * file_type_size)))
BAIL(NC_ENOMEM);
}
else
if (!bufr)
bufr = data;

/* Create the data transfer property list. */
if ((xfer_plistid = H5Pcreate(H5P_DATASET_XFER)) < 0)
BAIL(NC_EHDFERR);
Expand All @@ -2047,23 +2047,6 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
((NC_HDF5_TYPE_INFO_T *)var->type_info->format_type_info)->native_hdf_typeid,
mem_spaceid, file_spaceid, xfer_plistid, bufr) < 0)
BAIL(NC_EHDFERR);

/* Convert data type if needed. */
if (need_to_convert)
{
if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
BAIL(retval);

/* For strict netcdf-3 rules, ignore erange errors between UBYTE
* and BYTE types. */
if ((h5->cmode & NC_CLASSIC_MODEL) &&
(var->type_info->hdr.id == NC_UBYTE || var->type_info->hdr.id == NC_BYTE) &&
(mem_nc_type == NC_UBYTE || mem_nc_type == NC_BYTE) &&
range_error)
range_error = 0;
}
} /* endif ! no_read */
else
{
Expand Down Expand Up @@ -2123,7 +2106,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
fill_len *= (fill_value_size[d2] ? fill_value_size[d2] : 1);

/* Copy the fill value into the rest of the data buffer. */
filldata = (char *)data + real_data_size;
filldata = (char *)bufr + real_data_size;
for (i = 0; i < fill_len; i++)
{

Expand All @@ -2149,9 +2132,26 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
else
memcpy(filldata, fillvalue, file_type_size);
filldata = (char *)filldata + file_type_size;
}
}
}

/* Convert data type if needed. */
if (need_to_convert)
{
if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
BAIL(retval);

/* For strict netcdf-3 rules, ignore erange errors between UBYTE
* and BYTE types. */
if ((h5->cmode & NC_CLASSIC_MODEL) &&
(var->type_info->hdr.id == NC_UBYTE || var->type_info->hdr.id == NC_BYTE) &&
(mem_nc_type == NC_UBYTE || mem_nc_type == NC_BYTE) &&
range_error)
range_error = 0;
}

exit:
if (file_spaceid > 0)
if (H5Sclose(file_spaceid) < 0)
Expand Down
97 changes: 49 additions & 48 deletions libnczarr/zvar.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1678,6 +1678,38 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
log_dim_info(var, fdims, fmaxdims, start, count);
#endif

/* Check the type_info fields. */
assert(var->type_info && var->type_info->size &&
var->type_info->format_type_info);

/* Later on, we will need to know the size of this type in the
* file. */
file_type_size = var->type_info->size;

/* Are we going to convert any data? (No converting of compound or
* opaque types.) */
if (mem_nc_type != var->type_info->hdr.id &&
mem_nc_type != NC_COMPOUND && mem_nc_type != NC_OPAQUE)
{
/* We must convert - allocate a buffer. */
need_to_convert++;
if (var->ndims)
for (d2 = 0; d2 < var->ndims; d2++)
len *= countp[d2];
LOG((4, "converting data for var %s type=%d len=%d", var->hdr.name,
var->type_info->hdr.id, len));

/* If we're reading, we need bufr to have enough memory to store
* the data in the file. If we're writing, we need bufr to be
* big enough to hold all the data in the file's type. */
if (len > 0)
if (!(bufr = malloc(len * file_type_size)))
BAIL(NC_ENOMEM);
}
else
if (!bufr)
bufr = data;

/* Check dimension bounds. Remember that unlimited dimensions can
* put data beyond their current length. */
for (d2 = 0; d2 < var->ndims; d2++)
Expand Down Expand Up @@ -1739,14 +1771,6 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
}
}

/* Check the type_info fields. */
assert(var->type_info && var->type_info->size &&
var->type_info->format_type_info);

/* Later on, we will need to know the size of this type in the
* file. */
file_type_size = var->type_info->size;

if (!no_read)
{
#ifdef LOOK
Expand Down Expand Up @@ -1792,29 +1816,6 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
bufr = *(char **)data;
}
#endif
/* Are we going to convert any data? (No converting of compound or
* opaque types.) */
if (mem_nc_type != var->type_info->hdr.id &&
mem_nc_type != NC_COMPOUND && mem_nc_type != NC_OPAQUE)
{
/* We must convert - allocate a buffer. */
need_to_convert++;
if (var->ndims)
for (d2 = 0; d2 < var->ndims; d2++)
len *= countp[d2];
LOG((4, "converting data for var %s type=%d len=%d", var->hdr.name,
var->type_info->hdr.id, len));

/* If we're reading, we need bufr to have enough memory to store
* the data in the file. If we're writing, we need bufr to be
* big enough to hold all the data in the file's type. */
if (len > 0)
if (!(bufr = malloc(len * file_type_size)))
BAIL(NC_ENOMEM);
}
else
if (!bufr)
bufr = data;

#ifdef LOOK
/* Create the data transfer property list. */
Expand All @@ -1831,24 +1832,8 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,

if((retval = NCZ_transferslice(var, READING, start, count, stride, bufr, var->type_info->hdr.id)))
BAIL(retval);

/* Convert data type if needed. */
if (need_to_convert)
{
if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
BAIL(retval);

/* For strict netcdf-3 rules, ignore erange errors between UBYTE
* and BYTE types. */
if ((h5->cmode & NC_CLASSIC_MODEL) &&
(var->type_info->hdr.id == NC_UBYTE || var->type_info->hdr.id == NC_BYTE) &&
(mem_nc_type == NC_UBYTE || mem_nc_type == NC_BYTE) &&
range_error)
range_error = 0;
}
} /* endif ! no_read */

/* Now we need to fake up any further data that was asked for,
using the fill values instead. First skip past the data we
just read, if any. */
Expand Down Expand Up @@ -1902,6 +1887,22 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
}
}

/* Convert data type if needed. */
if (need_to_convert)
{
if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
BAIL(retval);
/* For strict netcdf-3 rules, ignore erange errors between UBYTE
* and BYTE types. */
if ((h5->cmode & NC_CLASSIC_MODEL) &&
(var->type_info->hdr.id == NC_UBYTE || var->type_info->hdr.id == NC_BYTE) &&
(mem_nc_type == NC_UBYTE || mem_nc_type == NC_BYTE) &&
range_error)
range_error = 0;
}

exit:
#ifdef LOOK
if (file_spaceid > 0)
Expand Down
2 changes: 2 additions & 0 deletions nc_test4/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ IF(BUILD_UTILITIES)
build_bin_test(renamegroup)
add_sh_test(nc_test4 run_grp_rename)
ADD_SH_TEST(nc_test4 tst_misc)
build_bin_test(tst_fillonly)
ADD_SH_TEST(nc_test4 test_fillonly)
IF(ENABLE_FILTER_TESTING)
build_bin_test(tst_filterparser)
build_bin_test(test_filter)
Expand Down
8 changes: 6 additions & 2 deletions nc_test4/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ if BUILD_UTILITIES
check_PROGRAMS += renamegroup
TESTS += run_grp_rename.sh tst_misc.sh

check_PROGRAMS += tst_fillonly
TESTS += test_fillonly.sh

# Szip Tests (requires ncdump)
if USE_SZIP
check_PROGRAMS += test_szip h5testszip
Expand Down Expand Up @@ -100,7 +103,8 @@ ref_szip.cdl tst_filter.sh bzip2.cdl ref_filtered.cdl \
ref_unfiltered.cdl ref_bzip2.c findplugin.in ref_unfilteredvv.cdl \
ref_filteredvv.cdl ref_multi.cdl \
ref_ncgenF.cdl ref_nccopyF.cdl \
ref_filter_order.txt ref_filter_repeat.txt
ref_filter_order.txt ref_filter_repeat.txt \
ref_fillonly.cdl test_fillonly.sh

CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc \
Expand All @@ -110,7 +114,7 @@ tst_*.h5 tst_grp_rename.cdl tst_grp_rename.dmp ref_grp_rename.cdl \
foo1.nc tst_*.h4 test.nc testszip.nc test.h5 szip_dump.cdl \
perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \
floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \
testfilter_reg.nc filterorder.txt filterrepeat.txt
testfilter_reg.nc filterorder.txt filterrepeat.txt tmp_fillonly.nc

DISTCLEANFILES = findplugin.sh run_par_test.sh

Expand Down
18 changes: 18 additions & 0 deletions nc_test4/ref_fillonly.cdl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
netcdf ref_fillonly {
dimensions:
x = UNLIMITED ; // (6 currently)
variables:
int i(x) ;
i:_Storage = "chunked" ;
i:_ChunkSizes = 6 ;
float f(x) ;
f:_FillValue = -9999.f ;
f:_Storage = "chunked" ;
f:_ChunkSizes = 6 ;

// global attributes:
:_Format = "netCDF-4" ;
data:

i = 1, 2, 3, 4, 5, 6 ;
}
Loading

0 comments on commit ff9d1b2

Please sign in to comment.