Skip to content

Commit 70a5e59

Browse files
vchoi-hdfgroupvchoi-hdfgroup
committed
Implementation for upper level APIs as documented in RFC:Programming model to Support Sparse Data in HDF5:
--APIs to handle sparse data --APIs to support direct chunk I/O --APIs for structured chunk filtering --test/sparse_storage.c to trigger the coding path for the APIs
1 parent 329207a commit 70a5e59

16 files changed

+3366
-6
lines changed

‎doxygen/examples/tables/propertyLists.dox

+20
Original file line numberDiff line numberDiff line change
@@ -905,6 +905,26 @@ encoding for object names.</td>
905905
<td>#H5Pset_fletcher32</td>
906906
<td>Sets up use of the Fletcher32 checksum filter.</td>
907907
</tr>
908+
<tr>
909+
<td>#H5Pset_filter2/#H5Pget_filter3</td>
910+
<td>Adds/gets a filter to/from the filter pipeline for a specified section of structured chunk.</td>
911+
</tr>
912+
<tr>
913+
<td>#H5Pget_filter_by_id3</td>
914+
<td>Returns information about a filter in the pipeline for a specified section of structured chunk.</td>
915+
</tr>
916+
<tr>
917+
<td>#H5Pget_nfilters2</td>
918+
<td>Returns the number of filters in the pipeline for a specified section of structured chunk.</td>
919+
</tr>
920+
<tr>
921+
<td>#H5Premove_filter2</td>
922+
<td>Remove a filter in the filter pipeline for a specified section of structured chunk.</td>
923+
</tr>
924+
<tr>
925+
<td>#H5Pmodify_filter2</td>
926+
<td>Modifies a filter in the filter pipeline for a specified section of structured chunk.</td>
927+
</tr>
908928
</table>
909929
//! [ocpl_table]
910930
*

‎src/H5D.c

+359
Large diffs are not rendered by default.

‎src/H5Dchunk.c

+149
Original file line numberDiff line numberDiff line change
@@ -8253,3 +8253,152 @@ H5D__chunk_get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t *of
82538253
done:
82548254
FUNC_LEAVE_NOAPI(ret_value)
82558255
} /* end H5D__chunk_get_offset_copy() */
8256+
8257+
/*-------------------------------------------------------------------------
8258+
* Function: H5D__write_struct_chunk_direct
8259+
*
8260+
* Purpose: Internal routine to write a structured chunk directly into the file.
8261+
*
8262+
* Return: Non-negative on success/Negative on failure
8263+
*
8264+
*-------------------------------------------------------------------------
8265+
*/
8266+
herr_t
8267+
H5D__write_struct_chunk_direct(H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset,
8268+
H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, void H5_ATTR_UNUSED *buf[])
8269+
{
8270+
herr_t ret_value = SUCCEED; /* Return value */
8271+
8272+
FUNC_ENTER_PACKAGE_NOERR
8273+
8274+
/* Sanity checks */
8275+
/* TBD: check for H5D_SPARSE_CHUNK */
8276+
/* assert(layout->type == H5D_SPARSE_CHUNK); */
8277+
8278+
/* TBD: set up and call routine to write the structured chunk */
8279+
/* FOR NOW: just return success */
8280+
8281+
FUNC_LEAVE_NOAPI(ret_value)
8282+
} /* end H5D__write_struct_chunk_direct() */
8283+
8284+
/*-------------------------------------------------------------------------
8285+
* Function: H5D__read_struct_chunk_direct
8286+
*
8287+
* Purpose: Internal routine to read a structured chunk directly from the file.
8288+
*
8289+
* Return: Non-negative on success/Negative on failure
8290+
*
8291+
*-------------------------------------------------------------------------
8292+
*/
8293+
herr_t
8294+
H5D__read_struct_chunk_direct(const H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset,
8295+
H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, void H5_ATTR_UNUSED *buf[])
8296+
{
8297+
herr_t ret_value = SUCCEED; /* Return value */
8298+
8299+
FUNC_ENTER_PACKAGE_NOERR
8300+
8301+
/* Check args */
8302+
/* TBD: check for H5D_SPARSE_CHUNK */
8303+
/* assert(dset && H5D_SPARSE_CHUNK == layout->type); */
8304+
assert(offset);
8305+
assert(chunk_info);
8306+
assert(buf);
8307+
8308+
/* TBD: set up and call routine to read the structured chunk */
8309+
/* FOR NOW: just return success */
8310+
8311+
FUNC_LEAVE_NOAPI(ret_value)
8312+
} /* end H5D__read_struct_chunk_direct() */
8313+
8314+
/*-------------------------------------------------------------------------
8315+
* Function: H5D__get_struct_chunk_info
8316+
*
8317+
* Purpose: Iterate over the chunks in the dataset to get the info
8318+
* of the desired chunk.
8319+
*
8320+
* TBD: Is the following also true for structured chunk?
8321+
* Note: Currently, the domain of the index in this function is of all
8322+
* the written chunks, regardless the dataspace.
8323+
*
8324+
* Return: SUCCEED/FAIL
8325+
*-------------------------------------------------------------------------
8326+
*/
8327+
herr_t
8328+
H5D__get_struct_chunk_info(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *space,
8329+
hsize_t H5_ATTR_UNUSED chunk_idx, hsize_t H5_ATTR_UNUSED *offset,
8330+
H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr,
8331+
hsize_t H5_ATTR_UNUSED *chunk_size)
8332+
{
8333+
herr_t ret_value = SUCCEED; /* Return value */
8334+
8335+
FUNC_ENTER_PACKAGE_NOERR
8336+
8337+
assert(dset);
8338+
assert(dset->shared);
8339+
assert(space);
8340+
8341+
/* TBD: go get structured chunk information using chunk index */
8342+
/* FOR NOW: just return success */
8343+
8344+
FUNC_LEAVE_NOAPI(ret_value)
8345+
} /* end H5D__get_struct_chunk_info() */
8346+
8347+
/*-------------------------------------------------------------------------
8348+
* Function: H5D__get_struct_chunk_info_by_coord
8349+
*
8350+
* Purpose: Iterate over the structured chunks in the dataset to get the info
8351+
* of the desired chunk, given by its offset coordinates.
8352+
*
8353+
* Return: Success: Non-negative
8354+
* Failure: Negative
8355+
*
8356+
*-------------------------------------------------------------------------
8357+
*/
8358+
herr_t
8359+
H5D__get_struct_chunk_info_by_coord(const H5D_t H5_ATTR_UNUSED *dset, const hsize_t H5_ATTR_UNUSED *offset,
8360+
H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr,
8361+
hsize_t H5_ATTR_UNUSED *chunk_size)
8362+
{
8363+
herr_t ret_value = SUCCEED; /* Return value */
8364+
8365+
FUNC_ENTER_PACKAGE_NOERR
8366+
8367+
/* Check args */
8368+
assert(dset);
8369+
assert(dset->shared);
8370+
assert(offset);
8371+
8372+
/* TBD: go get structured chunk information using chunk coordinates */
8373+
/* FOR NOW: just return success */
8374+
8375+
FUNC_LEAVE_NOAPI(ret_value)
8376+
} /* end H5D__get_struct_chunk_info_by_coord() */
8377+
8378+
/*-------------------------------------------------------------------------
8379+
* Function: H5D__struct_chunk_iter
8380+
*
8381+
* Purpose: Iterate over all the structured chunks in the dataset
8382+
* with given callback and the callback's required data.
8383+
*
8384+
* Return: SUCCEED/FAIL
8385+
*-------------------------------------------------------------------------
8386+
*/
8387+
herr_t
8388+
H5D__struct_chunk_iter(H5D_t H5_ATTR_UNUSED *dset, H5D_struct_chunk_iter_op_t H5_ATTR_UNUSED op,
8389+
void H5_ATTR_UNUSED *op_data)
8390+
{
8391+
herr_t ret_value = SUCCEED; /* Return value */
8392+
8393+
FUNC_ENTER_PACKAGE_NOERR
8394+
8395+
/* Check args */
8396+
assert(dset);
8397+
assert(dset->shared);
8398+
8399+
/* TBD: iterate over all the structured chunks in the dataset */
8400+
/* FOR NOW: just return success */
8401+
8402+
FUNC_LEAVE_NOAPI(ret_value)
8403+
} /* end H5D__chunk_iter() */
8404+

‎src/H5Dint.c

+72
Original file line numberDiff line numberDiff line change
@@ -4010,3 +4010,75 @@ H5D_get_dcpl_id(const H5D_obj_create_t *d)
40104010

40114011
FUNC_LEAVE_NOAPI(d->dcpl_id);
40124012
} /* end H5D_get_dcpl_id() */
4013+
4014+
/*-------------------------------------------------------------------------
4015+
* Function: H5D__get_defined
4016+
*
4017+
* Purpose: Returns the dataspace ID with selection of defined elements
4018+
*
4019+
* Return: Success: ID for dataspace
4020+
* Failure: FAIL
4021+
*
4022+
*-------------------------------------------------------------------------
4023+
*/
4024+
hid_t
4025+
H5D__get_defined(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t *fspace)
4026+
{
4027+
H5S_t *space = NULL;
4028+
hid_t ret_value = H5I_INVALID_HID;
4029+
4030+
FUNC_ENTER_PACKAGE
4031+
4032+
/* TBD:
4033+
if (dset->shared->layout.type == H5D_SPARSE_CHUNK)
4034+
call routine to get defined elements
4035+
else
4036+
if (NULL == (space = H5S_copy(fspace, false, true)))
4037+
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace");
4038+
*/
4039+
4040+
/* FOR NOW: return copy of fspace */
4041+
if (NULL == (space = H5S_copy(fspace, false, true)))
4042+
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace");
4043+
4044+
/* Create an ID */
4045+
if ((ret_value = H5I_register(H5I_DATASPACE, space, true)) < 0)
4046+
HGOTO_ERROR(H5E_ID, H5E_CANTREGISTER, FAIL, "unable to register dataspace");
4047+
4048+
done:
4049+
if (ret_value < 0)
4050+
if (space != NULL)
4051+
if (H5S_close(space) < 0)
4052+
HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release dataspace");
4053+
4054+
FUNC_LEAVE_NOAPI(ret_value)
4055+
4056+
} /* H5D__get_defined() */
4057+
4058+
/*-------------------------------------------------------------------------
4059+
*
4060+
* Function: H5D__erase
4061+
*
4062+
* Purpose: Deletes elements from a dataset
4063+
*
4064+
* Return: SUCCEED/FAIL
4065+
*
4066+
*-------------------------------------------------------------------------
4067+
*/
4068+
herr_t
4069+
H5D__erase(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *fspace)
4070+
{
4071+
herr_t ret_value = SUCCEED; /* Return value */
4072+
4073+
FUNC_ENTER_PACKAGE_NOERR
4074+
4075+
/* TBD:
4076+
if (dset->shared->layout.type == H5D_SPARSE_CHUNK)
4077+
call routine to delete elements
4078+
else
4079+
return error
4080+
*/
4081+
/* FOR NOW: just return success */
4082+
4083+
FUNC_LEAVE_NOAPI(ret_value)
4084+
} /* H5D__erase() */

‎src/H5Dpkg.h

+11
Original file line numberDiff line numberDiff line change
@@ -755,6 +755,17 @@ H5_DLL herr_t H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_
755755
H5_DLL herr_t H5D__chunk_stats(const H5D_t *dset, bool headers);
756756
#endif /* H5D_CHUNK_DEBUG */
757757

758+
/* Functions that operate on H5D_SPARSE_CHUNK storage */
759+
H5_DLL hid_t H5D__get_defined(const H5D_t *dset, const H5S_t *fspace);
760+
H5_DLL herr_t H5D__erase(const H5D_t *dset, const H5S_t *fspace);
761+
H5_DLL herr_t H5D__write_struct_chunk_direct(H5D_t *dset, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]);
762+
H5_DLL herr_t H5D__read_struct_chunk_direct(const H5D_t *dset, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]);
763+
H5_DLL herr_t H5D__get_struct_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t chunk_idx, hsize_t *offset,
764+
H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size);
765+
H5_DLL herr_t H5D__get_struct_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info,
766+
haddr_t *addr, hsize_t *chunk_size);
767+
H5_DLL herr_t H5D__struct_chunk_iter(H5D_t *dset, H5D_struct_chunk_iter_op_t cb, void *op_data);
768+
758769
/* format convert */
759770
H5_DLL herr_t H5D__chunk_format_convert(H5D_t *dset, H5D_chk_idx_info_t *idx_info,
760771
H5D_chk_idx_info_t *new_idx_info);

0 commit comments

Comments
 (0)