From 3eb9bc1c1dc6d59fd34b8883782c6f9963de191c Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Jun 2024 12:01:21 -0700 Subject: [PATCH 01/10] Remove ROS3_DEBUG printf debugging scheme --- src/H5FDros3.c | 87 ++------------------------------------------------ 1 file changed, 2 insertions(+), 85 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index d3fcd1536af..05fd419018d 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -35,14 +35,7 @@ #ifdef H5_HAVE_ROS3_VFD -/* Set debug output level */ -#define ROS3_DEBUG_NONE 0 -#define ROS3_DEBUG_TRACE_API 1 -#define ROS3_DEBUG_TRACE_INTERNAL 2 -#define ROS3_DEBUG ROS3_DEBUG_NONE - -/* toggle stats collection and reporting - */ +/* Toggle stats collection and reporting */ #define ROS3_STATS 0 /* Max size of the cache, in bytes */ @@ -283,10 +276,6 @@ H5FD_ros3_init(void) FUNC_ENTER_NOAPI(H5I_INVALID_HID) -#if ROS3_DEBUG - fprintf(stdout, "H5FD_ros3_init() called.\n"); -#endif - if (H5I_VFL != H5I_get_type(H5FD_ROS3_g)) { H5FD_ROS3_g = H5FD_register(&H5FD_ros3_g, sizeof(H5FD_class_t), false); if (H5I_INVALID_HID == H5FD_ROS3_g) { @@ -323,10 +312,6 @@ H5FD__ros3_term(void) { FUNC_ENTER_PACKAGE_NOERR -#if ROS3_DEBUG - fprintf(stdout, "H5FD__ros3_term() called.\n"); -#endif - /* Reset VFL ID */ H5FD_ROS3_g = 0; @@ -352,10 +337,6 @@ H5Pset_fapl_ros3(hid_t fapl_id, const H5FD_ros3_fapl_t *fa) assert(fa != NULL); -#if ROS3_DEBUG - fprintf(stdout, "H5Pset_fapl_ros3() called.\n"); -#endif - plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); if (plist == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list"); @@ -416,10 +397,6 @@ H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_dst /*out*/) FUNC_ENTER_API(FAIL) -#if ROS3_DEBUG - fprintf(stdout, "H5Pget_fapl_ros3() called.\n"); -#endif - if (fa_dst == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_dst is NULL"); @@ -553,10 +530,6 @@ H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/) FUNC_ENTER_API(FAIL) -#if ROS3_DEBUG - fprintf(stdout, "H5Pget_fapl_ros3_token() called.\n"); -#endif - if (size == 0) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "size cannot be zero."); if (token_dst == NULL) @@ -601,10 +574,6 @@ H5FD__ros3_str_token_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED FUNC_ENTER_PACKAGE -#if ROS3_DEBUG - fprintf(stdout, "H5FD__ros3_str_token_copy() called.\n"); -#endif - if (*value) if (NULL == (*value = strdup(*value))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't copy string property token"); @@ -716,10 +685,6 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) FUNC_ENTER_API(FAIL) -#if ROS3_DEBUG - fprintf(stdout, "H5Pset_fapl_ros3_token() called.\n"); -#endif - if (fapl_id == H5P_DEFAULT) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list"); if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) @@ -771,10 +736,6 @@ ros3_reset_stats(H5FD_ros3_t *file) FUNC_ENTER_PACKAGE -#if ROS3_DEBUG >= ROS3_DEBUG_TRACE_INTERNAL - printf("ros3_reset_stats() called\n"); -#endif - if (file == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file was null"); @@ -830,10 +791,6 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) FUNC_ENTER_PACKAGE -#if ROS3_DEBUG - fprintf(stdout, "H5FD__ros3_open() called.\n"); -#endif - /* Sanity check on file offsets */ HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t)); @@ -1223,10 +1180,6 @@ H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file) FUNC_ENTER_PACKAGE -#if ROS3_DEBUG - fprintf(stdout, "H5FD__ros3_close() called.\n"); -#endif - assert(file != NULL); assert(file->s3r_handle != NULL); @@ -1283,10 +1236,6 @@ H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2) FUNC_ENTER_PACKAGE_NOERR -#if ROS3_DEBUG - fprintf(stdout, "H5FD__ros3_cmp() called.\n"); -#endif - assert(f1->s3r_handle != NULL); assert(f2->s3r_handle != NULL); @@ -1385,10 +1334,6 @@ H5FD__ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) { FUNC_ENTER_PACKAGE_NOERR -#if ROS3_DEBUG >= ROS3_DEBUG_TRACE_INTERNAL - fprintf(stdout, "H5FD__ros3_query() called.\n"); -#endif - /* Set the VFL feature flags that this driver supports * * Since the ros3 VFD is read-only, many flags are irrelevant. @@ -1419,10 +1364,6 @@ H5FD__ros3_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) FUNC_ENTER_PACKAGE_NOERR -#if ROS3_DEBUG >= ROS3_DEBUG_TRACE_INTERNAL - fprintf(stdout, "H5FD__ros3_get_eoa() called.\n"); -#endif - FUNC_LEAVE_NOAPI(file->eoa) } /* end H5FD__ros3_get_eoa() */ @@ -1441,10 +1382,6 @@ H5FD__ros3_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) FUNC_ENTER_PACKAGE_NOERR -#if ROS3_DEBUG >= ROS3_DEBUG_TRACE_INTERNAL - fprintf(stdout, "H5FD__ros3_set_eoa() called.\n"); -#endif - file->eoa = addr; FUNC_LEAVE_NOAPI(SUCCEED) @@ -1466,10 +1403,6 @@ H5FD__ros3_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) FUNC_ENTER_PACKAGE_NOERR -#if ROS3_DEBUG >= ROS3_DEBUG_TRACE_INTERNAL - fprintf(stdout, "H5FD__ros3_get_eof() called.\n"); -#endif - FUNC_LEAVE_NOAPI(H5FD_s3comms_s3r_get_filesize(file->s3r_handle)) } /* end H5FD__ros3_get_eof() */ @@ -1489,10 +1422,6 @@ H5FD__ros3_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_hand FUNC_ENTER_PACKAGE -#if ROS3_DEBUG - fprintf(stdout, "H5FD__ros3_get_handle() called.\n"); -#endif - if (!file_handle) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid"); @@ -1520,17 +1449,13 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU size_t filesize = 0; herr_t ret_value = SUCCEED; #if ROS3_STATS - /* working variables for storing stats */ + /* Working variables for storing stats */ ros3_statsbin *bin = NULL; unsigned bin_i = 0; #endif FUNC_ENTER_PACKAGE -#if ROS3_DEBUG - fprintf(stdout, "H5FD__ros3_read() called.\n"); -#endif - assert(file); assert(file->cache); assert(file->s3r_handle); @@ -1596,10 +1521,6 @@ H5FD__ros3_write(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, h FUNC_ENTER_PACKAGE -#if ROS3_DEBUG - fprintf(stdout, "H5FD__ros3_write() called.\n"); -#endif - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot write to read-only file."); done: @@ -1624,10 +1545,6 @@ H5FD__ros3_truncate(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED dxpl_id, FUNC_ENTER_PACKAGE -#if ROS3_DEBUG - fprintf(stdout, "H5FD__ros3_truncate() called.\n"); -#endif - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot truncate read-only file."); done: From ce4b13db8a333708d707dda18cd6d1b40d621d88 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Jun 2024 13:04:52 -0700 Subject: [PATCH 02/10] Clean up ros3 VFD stats * Move ROS3_STATS code around * Add _g to ros3_stats_boundaries global variable * Add H5FD__ prefix to stats functions to avoid FUNC_ENTER errors --- src/H5FDros3.c | 55 ++++++++++++++++++++++---------------------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 05fd419018d..aea1f0c88a7 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -36,7 +36,7 @@ #ifdef H5_HAVE_ROS3_VFD /* Toggle stats collection and reporting */ -#define ROS3_STATS 0 +#define ROS3_STATS 1 /* Max size of the cache, in bytes */ #define ROS3_MAX_CACHE_SIZE 16777216 @@ -75,7 +75,7 @@ static hid_t H5FD_ROS3_g = 0; #define ROS3_STATS_POW(bin_i, out_ptr) \ { \ unsigned long long donotshadowresult = 1; \ - unsigned donotshadowindex = 0; \ + int donotshadowindex = 0; \ for (donotshadowindex = 0; \ donotshadowindex < (((bin_i)*ROS3_STATS_INTERVAL) + ROS3_STATS_START_POWER); \ donotshadowindex++) { \ @@ -85,7 +85,7 @@ static hid_t H5FD_ROS3_g = 0; } /* Array to hold pre-computed boundaries for stats bins */ -static unsigned long long ros3_stats_boundaries[ROS3_STATS_BIN_COUNT]; +static unsigned long long ros3_stats_boundaries_g[ROS3_STATS_BIN_COUNT]; /*************************************************************************** * Structure for storing per-file ros3 VFD usage statistics. @@ -270,9 +270,6 @@ hid_t H5FD_ros3_init(void) { hid_t ret_value = H5I_INVALID_HID; -#if ROS3_STATS - unsigned int bin_i; -#endif FUNC_ENTER_NOAPI(H5I_INVALID_HID) @@ -284,11 +281,11 @@ H5FD_ros3_init(void) #if ROS3_STATS /* Pre-compute statsbin boundaries */ - for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) { + for (int i = 0; i < ROS3_STATS_BIN_COUNT; i++) { unsigned long long value = 0; - ROS3_STATS_POW(bin_i, &value) - ros3_stats_boundaries[bin_i] = value; + ROS3_STATS_POW(i, &value) + ros3_stats_boundaries_g[i] = value; } #endif } @@ -721,7 +718,7 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) #if ROS3_STATS /*---------------------------------------------------------------------------- - * Function: ros3_reset_stats + * Function: H5FD__ros3_reset_stats * * Purpose: Reset the collected statistics * @@ -729,17 +726,16 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) *---------------------------------------------------------------------------- */ static herr_t -ros3_reset_stats(H5FD_ros3_t *file) +H5FD__ros3_reset_stats(H5FD_ros3_t *file) { - unsigned i = 0; - herr_t ret_value = SUCCEED; + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE if (file == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file was null"); - for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { + for (int i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { file->raw[i].bytes = 0; file->raw[i].count = 0; file->raw[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN; @@ -753,7 +749,7 @@ ros3_reset_stats(H5FD_ros3_t *file) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end ros3_reset_stats() */ +} /* end H5FD__ros3_reset_stats() */ #endif /* ROS3_STATS */ /*------------------------------------------------------------------------- @@ -863,7 +859,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) H5MM_memcpy(&(file->fa), &fa, sizeof(H5FD_ros3_fapl_t)); #if ROS3_STATS - if (FAIL == ros3_reset_stats(file)) + if (FAIL == H5FD__ros3_reset_stats(file)) HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics"); #endif @@ -898,7 +894,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) #if ROS3_STATS /*---------------------------------------------------------------------------- - * Function: ros3_fprint_stats + * Function: H5FD__ros3_fprint_stats * * Purpose: Tabulate and pretty-print statistics for this virtual file. * @@ -941,7 +937,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) *---------------------------------------------------------------------------- */ static herr_t -ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) +H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) { herr_t ret_value = SUCCEED; parsed_url_t *purl = NULL; @@ -1106,10 +1102,10 @@ ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) if (r->count == 0 && m->count == 0) continue; - range_end = ros3_stats_boundaries[i]; + range_end = ros3_stats_boundaries_g[i]; if (i == ROS3_STATS_BIN_COUNT) { - range_end = ros3_stats_boundaries[i - 1]; + range_end = ros3_stats_boundaries_g[i - 1]; fprintf(stream, ">"); } else @@ -1161,7 +1157,7 @@ ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) done: FUNC_LEAVE_NOAPI(ret_value) -} /* ros3_fprint_stats */ +} /* H5FD__ros3_fprint_stats */ #endif /* ROS3_STATS */ /*------------------------------------------------------------------------- @@ -1184,8 +1180,7 @@ H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file) assert(file->s3r_handle != NULL); #if ROS3_STATS - /* TODO: mechanism to re-target stats printout */ - if (ros3_fprint_stats(stdout, file) == FAIL) + if (H5FD__ros3_fprint_stats(stdout, file) == FAIL) HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics"); #endif @@ -1448,11 +1443,6 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU H5FD_ros3_t *file = (H5FD_ros3_t *)_file; size_t filesize = 0; herr_t ret_value = SUCCEED; -#if ROS3_STATS - /* Working variables for storing stats */ - ros3_statsbin *bin = NULL; - unsigned bin_i = 0; -#endif FUNC_ENTER_PACKAGE @@ -1477,11 +1467,14 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read"); #if ROS3_STATS + ros3_statsbin *bin = NULL; + int i = 0; + /* Find which "bin" this read fits in. Can be "overflow" bin. */ - for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) - if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) + for (i = 0; i < ROS3_STATS_BIN_COUNT; i++) + if ((unsigned long long)size < ros3_stats_boundaries_g[i]) break; - bin = (type == H5FD_MEM_DRAW) ? &file->raw[bin_i] : &file->meta[bin_i]; + bin = (type == H5FD_MEM_DRAW) ? &file->raw[i] : &file->meta[i]; /* Store collected stats in appropriate bin */ if (bin->count == 0) { From 7b6f5856a4f79a7e3d5b307a4586f9b8737f6dc0 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Jun 2024 13:11:40 -0700 Subject: [PATCH 03/10] Switch #if ROS3_STATS to #ifdef ROS3_STATS --- src/H5FDros3.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index aea1f0c88a7..f09742c3238 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -35,8 +35,8 @@ #ifdef H5_HAVE_ROS3_VFD -/* Toggle stats collection and reporting */ -#define ROS3_STATS 1 +/* Define to turn on stats collection and reporting */ +/* #define ROS3_STATS */ /* Max size of the cache, in bytes */ #define ROS3_MAX_CACHE_SIZE 16777216 @@ -47,7 +47,7 @@ static hid_t H5FD_ROS3_g = 0; /* Session/security token property name */ #define ROS3_TOKEN_PROP_NAME "ros3_token_prop" -#if ROS3_STATS +#ifdef ROS3_STATS /* Arbitrarily large value, such that any reasonable size read will be "less" * than this value and set a true minimum @@ -166,7 +166,7 @@ typedef struct H5FD_ros3_t { s3r_t *s3r_handle; uint8_t *cache; size_t cache_size; -#if ROS3_STATS +#ifdef ROS3_STATS ros3_statsbin meta[ROS3_STATS_BIN_COUNT + 1]; ros3_statsbin raw[ROS3_STATS_BIN_COUNT + 1]; #endif @@ -279,7 +279,7 @@ H5FD_ros3_init(void) HGOTO_ERROR(H5E_ID, H5E_CANTREGISTER, H5I_INVALID_HID, "unable to register ros3"); } -#if ROS3_STATS +#ifdef ROS3_STATS /* Pre-compute statsbin boundaries */ for (int i = 0; i < ROS3_STATS_BIN_COUNT; i++) { unsigned long long value = 0; @@ -716,7 +716,7 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) FUNC_LEAVE_API(ret_value) } /* end H5Pset_fapl_ros3_token() */ -#if ROS3_STATS +#ifdef ROS3_STATS /*---------------------------------------------------------------------------- * Function: H5FD__ros3_reset_stats * @@ -858,7 +858,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) file->s3r_handle = handle; H5MM_memcpy(&(file->fa), &fa, sizeof(H5FD_ros3_fapl_t)); -#if ROS3_STATS +#ifdef ROS3_STATS if (FAIL == H5FD__ros3_reset_stats(file)) HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics"); #endif @@ -892,7 +892,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__ros3_open() */ -#if ROS3_STATS +#ifdef ROS3_STATS /*---------------------------------------------------------------------------- * Function: H5FD__ros3_fprint_stats * @@ -1179,7 +1179,7 @@ H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file) assert(file != NULL); assert(file->s3r_handle != NULL); -#if ROS3_STATS +#ifdef ROS3_STATS if (H5FD__ros3_fprint_stats(stdout, file) == FAIL) HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics"); #endif @@ -1466,7 +1466,7 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read"); -#if ROS3_STATS +#ifdef ROS3_STATS ros3_statsbin *bin = NULL; int i = 0; From 5339a010ad4a3d705b9f0d748a8a815868e9f758 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Jun 2024 13:16:59 -0700 Subject: [PATCH 04/10] ros3_statsbin --> H5FD_ros3_stats_bin_t --- src/H5FDros3.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index f09742c3238..d9a8730d9bf 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -106,12 +106,12 @@ static unsigned long long ros3_stats_boundaries_g[ROS3_STATS_BIN_COUNT]; * * Largest read size in this bin. ***************************************************************************/ -typedef struct { +typedef struct H5FD_ros3_stats_bin { unsigned long long count; unsigned long long bytes; unsigned long long min; unsigned long long max; -} ros3_statsbin; +} H5FD_ros3_stats_bin_t; #endif /* ROS3_STATS */ @@ -141,14 +141,13 @@ typedef struct { * Responsible for communicating with remote host and presenting file * contents as indistinguishable from a file on the local filesystem. * - * *** present only if ROS3_SATS is flagged to enable stats collection *** + * *** present only if ROS3_SATS is set to enable stats collection *** * - * `meta` (ros3_statsbin[]) - * `raw` (ros3_statsbin[]) + * `meta` (H5FD_ros3_stats_bin_t[]) + * `raw` (H5FD_ros3_stats_bin_t[]) * - * Only present if ros3 stats collection is enabled. - * - * Arrays of `ros3_statsbin` structures to record raw- and metadata reads. + * Arrays of `H5FD_ros3_stats_bin_t` structures to record raw- and + * metadata reads. * * Records count and size of reads performed by the VFD, and is used to * print formatted usage statistics to stdout upon VFD shutdown. @@ -167,8 +166,8 @@ typedef struct H5FD_ros3_t { uint8_t *cache; size_t cache_size; #ifdef ROS3_STATS - ros3_statsbin meta[ROS3_STATS_BIN_COUNT + 1]; - ros3_statsbin raw[ROS3_STATS_BIN_COUNT + 1]; + H5FD_ros3_stats_bin_t meta[ROS3_STATS_BIN_COUNT + 1]; + H5FD_ros3_stats_bin_t raw[ROS3_STATS_BIN_COUNT + 1]; #endif } H5FD_ros3_t; @@ -992,8 +991,8 @@ H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) *******************/ for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { - const ros3_statsbin *r = &file->raw[i]; - const ros3_statsbin *m = &file->meta[i]; + const H5FD_ros3_stats_bin_t *r = &file->raw[i]; + const H5FD_ros3_stats_bin_t *m = &file->meta[i]; if (m->min < min_meta) min_meta = m->min; @@ -1085,8 +1084,8 @@ H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) fprintf(stream, " up-to meta raw meta raw meta raw\n"); for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { - const ros3_statsbin *m; - const ros3_statsbin *r; + const H5FD_ros3_stats_bin_t *m; + const H5FD_ros3_stats_bin_t *r; unsigned long long range_end = 0; char bm_suffix = ' '; /* bytes-meta */ double bm_val = 0.0; @@ -1467,7 +1466,7 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read"); #ifdef ROS3_STATS - ros3_statsbin *bin = NULL; + H5FD_ros3_stats_bin_t *bin = NULL; int i = 0; /* Find which "bin" this read fits in. Can be "overflow" bin. */ From c269904b359b10aad00f85050ac3b41a67d94120 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Jun 2024 13:18:03 -0700 Subject: [PATCH 05/10] Format source --- src/H5FDros3.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index d9a8730d9bf..98638b22f8f 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -1086,15 +1086,15 @@ H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { const H5FD_ros3_stats_bin_t *m; const H5FD_ros3_stats_bin_t *r; - unsigned long long range_end = 0; - char bm_suffix = ' '; /* bytes-meta */ - double bm_val = 0.0; - char br_suffix = ' '; /* bytes-raw */ - double br_val = 0.0; - char am_suffix = ' '; /* average-meta */ - double am_val = 0.0; - char ar_suffix = ' '; /* average-raw */ - double ar_val = 0.0; + unsigned long long range_end = 0; + char bm_suffix = ' '; /* bytes-meta */ + double bm_val = 0.0; + char br_suffix = ' '; /* bytes-raw */ + double br_val = 0.0; + char am_suffix = ' '; /* average-meta */ + double am_val = 0.0; + char ar_suffix = ' '; /* average-raw */ + double ar_val = 0.0; m = &file->meta[i]; r = &file->raw[i]; @@ -1467,7 +1467,7 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU #ifdef ROS3_STATS H5FD_ros3_stats_bin_t *bin = NULL; - int i = 0; + int i = 0; /* Find which "bin" this read fits in. Can be "overflow" bin. */ for (i = 0; i < ROS3_STATS_BIN_COUNT; i++) From 476239cf1f91586e4267dff6ed04eabf9cd8664b Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Jun 2024 13:35:17 -0700 Subject: [PATCH 06/10] Switch ros3 stat type to uint64_t --- src/H5FDros3.c | 86 ++++++++++++++++++++------------------------------ 1 file changed, 34 insertions(+), 52 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 98638b22f8f..f0b06186661 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -36,7 +36,7 @@ #ifdef H5_HAVE_ROS3_VFD /* Define to turn on stats collection and reporting */ -/* #define ROS3_STATS */ +#define ROS3_STATS /* Max size of the cache, in bytes */ #define ROS3_MAX_CACHE_SIZE 16777216 @@ -74,8 +74,8 @@ static hid_t H5FD_ROS3_g = 0; */ #define ROS3_STATS_POW(bin_i, out_ptr) \ { \ - unsigned long long donotshadowresult = 1; \ - int donotshadowindex = 0; \ + uint64_t donotshadowresult = 1; \ + int donotshadowindex = 0; \ for (donotshadowindex = 0; \ donotshadowindex < (((bin_i)*ROS3_STATS_INTERVAL) + ROS3_STATS_START_POWER); \ donotshadowindex++) { \ @@ -85,32 +85,14 @@ static hid_t H5FD_ROS3_g = 0; } /* Array to hold pre-computed boundaries for stats bins */ -static unsigned long long ros3_stats_boundaries_g[ROS3_STATS_BIN_COUNT]; +static uint64_t ros3_stats_boundaries_g[ROS3_STATS_BIN_COUNT]; -/*************************************************************************** - * Structure for storing per-file ros3 VFD usage statistics. - * - * `count` (unsigned long long) - * - * Number of reads with size in this bin's range. - * - * `bytes` (unsigned long long) - * - * Total number of bytes read through this bin. - * - * `min` (unsigned long long) - * - * Smallest read size in this bin. - * - * `max` (unsigned long long) - * - * Largest read size in this bin. - ***************************************************************************/ +/* Structure for storing per-file usage statistics */ typedef struct H5FD_ros3_stats_bin { - unsigned long long count; - unsigned long long bytes; - unsigned long long min; - unsigned long long max; + uint64_t count; /* # of reads with size in this bin's range */ + uint64_t bytes; /* Total bytes read in this bin */ + uint64_t min; /* Smallest read size in this bin */ + uint64_t max; /* Largest read size in this bin */ } H5FD_ros3_stats_bin_t; #endif /* ROS3_STATS */ @@ -281,7 +263,7 @@ H5FD_ros3_init(void) #ifdef ROS3_STATS /* Pre-compute statsbin boundaries */ for (int i = 0; i < ROS3_STATS_BIN_COUNT; i++) { - unsigned long long value = 0; + uint64_t value = 0; ROS3_STATS_POW(i, &value) ros3_stats_boundaries_g[i] = value; @@ -737,12 +719,12 @@ H5FD__ros3_reset_stats(H5FD_ros3_t *file) for (int i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { file->raw[i].bytes = 0; file->raw[i].count = 0; - file->raw[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN; + file->raw[i].min = (uint64_t)ROS3_STATS_STARTING_MIN; file->raw[i].max = 0; file->meta[i].bytes = 0; file->meta[i].count = 0; - file->meta[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN; + file->meta[i].min = (uint64_t)ROS3_STATS_STARTING_MIN; file->meta[i].max = 0; } @@ -938,22 +920,22 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) static herr_t H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) { - herr_t ret_value = SUCCEED; - parsed_url_t *purl = NULL; - unsigned i = 0; - unsigned long count_meta = 0; - unsigned long count_raw = 0; - double average_meta = 0.0; - double average_raw = 0.0; - unsigned long long min_meta = (unsigned long long)ROS3_STATS_STARTING_MIN; - unsigned long long min_raw = (unsigned long long)ROS3_STATS_STARTING_MIN; - unsigned long long max_meta = 0; - unsigned long long max_raw = 0; - unsigned long long bytes_raw = 0; - unsigned long long bytes_meta = 0; - double re_dub = 0.0; /* reusable double variable */ - unsigned suffix_i = 0; - const char suffixes[] = {' ', 'K', 'M', 'G', 'T', 'P'}; + herr_t ret_value = SUCCEED; + parsed_url_t *purl = NULL; + unsigned i = 0; + unsigned long count_meta = 0; + unsigned long count_raw = 0; + double average_meta = 0.0; + double average_raw = 0.0; + uint64_t min_meta = (uint64_t)ROS3_STATS_STARTING_MIN; + uint64_t min_raw = (uint64_t)ROS3_STATS_STARTING_MIN; + uint64_t max_meta = 0; + uint64_t max_raw = 0; + uint64_t bytes_raw = 0; + uint64_t bytes_meta = 0; + double re_dub = 0.0; /* reusable double variable */ + unsigned suffix_i = 0; + const char suffixes[] = {' ', 'K', 'M', 'G', 'T', 'P'}; FUNC_ENTER_PACKAGE @@ -1018,8 +1000,8 @@ H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) ******************/ fprintf(stream, "TOTAL READS: %lu (%lu meta, %lu raw)\n", count_raw + count_meta, count_meta, count_raw); - fprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n", bytes_raw + bytes_meta, bytes_meta, - bytes_raw); + fprintf(stream, "TOTAL BYTES: %" PRIu64 " (%" PRIu64 " meta, %" PRIu64 " raw)\n", bytes_raw + bytes_meta, + bytes_meta, bytes_raw); if (count_raw + count_meta == 0) goto done; @@ -1086,7 +1068,7 @@ H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { const H5FD_ros3_stats_bin_t *m; const H5FD_ros3_stats_bin_t *r; - unsigned long long range_end = 0; + uint64_t range_end = 0; char bm_suffix = ' '; /* bytes-meta */ double bm_val = 0.0; char br_suffix = ' '; /* bytes-raw */ @@ -1141,7 +1123,7 @@ H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) re_dub /= 1024.0; assert(suffix_i < sizeof(suffixes)); - fprintf(stream, " %8.3f%c %7llu %7llu %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", re_dub, + fprintf(stream, " %8.3f%c %7" PRIu64 " %7" PRIu64 " %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", re_dub, suffixes[suffix_i], /* Bin ceiling */ m->count, /* Metadata reads */ r->count, /* Raw data reads */ @@ -1471,7 +1453,7 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU /* Find which "bin" this read fits in. Can be "overflow" bin. */ for (i = 0; i < ROS3_STATS_BIN_COUNT; i++) - if ((unsigned long long)size < ros3_stats_boundaries_g[i]) + if ((uint64_t)size < ros3_stats_boundaries_g[i]) break; bin = (type == H5FD_MEM_DRAW) ? &file->raw[i] : &file->meta[i]; @@ -1487,7 +1469,7 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU bin->max = size; } bin->count++; - bin->bytes += (unsigned long long)size; + bin->bytes += (uint64_t)size; #endif } From 8ae8b98762608a6701334cfb7e2b98a4fc660c54 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Jun 2024 15:30:17 -0700 Subject: [PATCH 07/10] Removed ROS3_STATS_POW() Replaced the over-engineered macro with a simple bit shift --- src/H5FDros3.c | 46 ++++++++++++---------------------------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index f0b06186661..675f89715f4 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -49,6 +49,13 @@ static hid_t H5FD_ROS3_g = 0; #ifdef ROS3_STATS +/* The ros3 VFD can collect some simple I/O stats on a per-file basis. These + * are stored in arrays of bins (one for data and one for metadata) in the the + * VFD's file structure. Each bin contains stats for I/O operations of a given + * I/O size range. The bin boundaries are kept in a global "bin boundaries" + * array that is initialized at VFD startup does not change. + */ + /* Arbitrarily large value, such that any reasonable size read will be "less" * than this value and set a true minimum * @@ -56,33 +63,8 @@ static hid_t H5FD_ROS3_g = 0; */ #define ROS3_STATS_STARTING_MIN 0xfffffffful -/* Configuration definitions for stats collection and breakdown - * - * 2^10 = 1024 - * Reads up to 1024 bytes (1 kB) fall in bin 0 - * 2^(10+(1*16)) = 2^26 = 64MB - * Reads of 64MB or greater fall in "overflow" bin[BIN_COUNT] - */ -#define ROS3_STATS_BASE 2 -#define ROS3_STATS_INTERVAL 1 -#define ROS3_STATS_START_POWER 10 -#define ROS3_STATS_BIN_COUNT 16 /* MUST BE GREATER THAN 0 */ - -/* Calculate `BASE ^ (START_POWER + (INTERVAL * bin_i))` - * Stores result at `(unsigned long long *) out_ptr`. - * Used in computing boundaries between stats bins. - */ -#define ROS3_STATS_POW(bin_i, out_ptr) \ - { \ - uint64_t donotshadowresult = 1; \ - int donotshadowindex = 0; \ - for (donotshadowindex = 0; \ - donotshadowindex < (((bin_i)*ROS3_STATS_INTERVAL) + ROS3_STATS_START_POWER); \ - donotshadowindex++) { \ - donotshadowresult *= ROS3_STATS_BASE; \ - } \ - *(out_ptr) = donotshadowresult; \ - } +/* Number of bins */ +#define ROS3_STATS_BIN_COUNT 16 /* Array to hold pre-computed boundaries for stats bins */ static uint64_t ros3_stats_boundaries_g[ROS3_STATS_BIN_COUNT]; @@ -261,13 +243,9 @@ H5FD_ros3_init(void) } #ifdef ROS3_STATS - /* Pre-compute statsbin boundaries */ - for (int i = 0; i < ROS3_STATS_BIN_COUNT; i++) { - uint64_t value = 0; - - ROS3_STATS_POW(i, &value) - ros3_stats_boundaries_g[i] = value; - } + /* Pre-compute stats bin boundaries on powers of 2 >= 10 */ + for (int i = 0; i < ROS3_STATS_BIN_COUNT; i++) + ros3_stats_boundaries_g[i] = 1 << (10 + i); #endif } From 0347aec5ea4baf6ee3ccf1511293546d8b96c79e Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Jun 2024 15:37:48 -0700 Subject: [PATCH 08/10] Removed ROS3_STATS_STARTING_MIN 0 is fine... --- src/H5FDros3.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 675f89715f4..2fce82c5335 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -56,13 +56,6 @@ static hid_t H5FD_ROS3_g = 0; * array that is initialized at VFD startup does not change. */ -/* Arbitrarily large value, such that any reasonable size read will be "less" - * than this value and set a true minimum - * - * Not 0 because that may be a valid recorded minimum in degenerate cases - */ -#define ROS3_STATS_STARTING_MIN 0xfffffffful - /* Number of bins */ #define ROS3_STATS_BIN_COUNT 16 @@ -697,12 +690,12 @@ H5FD__ros3_reset_stats(H5FD_ros3_t *file) for (int i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { file->raw[i].bytes = 0; file->raw[i].count = 0; - file->raw[i].min = (uint64_t)ROS3_STATS_STARTING_MIN; + file->raw[i].min = 0; file->raw[i].max = 0; file->meta[i].bytes = 0; file->meta[i].count = 0; - file->meta[i].min = (uint64_t)ROS3_STATS_STARTING_MIN; + file->meta[i].min = 0; file->meta[i].max = 0; } @@ -905,8 +898,8 @@ H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) unsigned long count_raw = 0; double average_meta = 0.0; double average_raw = 0.0; - uint64_t min_meta = (uint64_t)ROS3_STATS_STARTING_MIN; - uint64_t min_raw = (uint64_t)ROS3_STATS_STARTING_MIN; + uint64_t min_meta = 0; + uint64_t min_raw = 0; uint64_t max_meta = 0; uint64_t max_raw = 0; uint64_t bytes_raw = 0; From eedf405d101351173a0a9f08e39e6c39820f95df Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Jun 2024 15:43:10 -0700 Subject: [PATCH 09/10] H5FD__ros3_fprint_stats --> H5FD__ros3_print_stats Also tidied some comments --- src/H5FDros3.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 2fce82c5335..fdff8498af1 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -53,7 +53,7 @@ static hid_t H5FD_ROS3_g = 0; * are stored in arrays of bins (one for data and one for metadata) in the the * VFD's file structure. Each bin contains stats for I/O operations of a given * I/O size range. The bin boundaries are kept in a global "bin boundaries" - * array that is initialized at VFD startup does not change. + * array that is initialized at VFD startup and does not change. */ /* Number of bins */ @@ -846,15 +846,14 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) #ifdef ROS3_STATS /*---------------------------------------------------------------------------- - * Function: H5FD__ros3_fprint_stats + * Function: H5FD__ros3_print_stats * * Purpose: Tabulate and pretty-print statistics for this virtual file. * * Should be called upon file close. * * Shows number of reads and bytes read, broken down by - * "raw" (H5FD_MEM_DRAW) - * or "meta" (any other flag) + * "raw" (H5FD_MEM_DRAW) or "meta" (any other flag) * * Prints filename and listing of total number of reads and bytes read, * both as a grand total and separate meta- and raw data reads. @@ -872,14 +871,12 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) * separate sub-colums for raw- and metadata reads. * - each row represents one bin, identified by the top of its range * - * Bin ranges can be modified with pound-defines at the top of this file. - * * Bins without any reads in their bounds are not printed. * * An "overflow" bin is also present, to catch "big" reads. * - * Output for all bins (and range ceiling and average size report) - * is divied by powers of 1024. By corollary, four digits before the decimal + * Output for all bins (and range ceiling and average size report) is + * divied by powers of 1024. By corollary, four digits before the decimal * is valid. * * - 41080 bytes is represented by 40.177k, not 41.080k @@ -889,7 +886,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) *---------------------------------------------------------------------------- */ static herr_t -H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) +H5FD__ros3_print_stats(FILE *stream, const H5FD_ros3_t *file) { herr_t ret_value = SUCCEED; parsed_url_t *purl = NULL; @@ -1109,7 +1106,7 @@ H5FD__ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__ros3_fprint_stats */ +} /* H5FD__ros3_print_stats */ #endif /* ROS3_STATS */ /*------------------------------------------------------------------------- @@ -1132,7 +1129,7 @@ H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file) assert(file->s3r_handle != NULL); #ifdef ROS3_STATS - if (H5FD__ros3_fprint_stats(stdout, file) == FAIL) + if (H5FD__ros3_print_stats(stdout, file) == FAIL) HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics"); #endif From 350faf765ea9467c725c81c16eb2cfe9f153efcd Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Tue, 18 Jun 2024 10:49:12 -0700 Subject: [PATCH 10/10] Move stats functions to a common area Moves the stats to a common ifdef block and adds an H5FD__ros3_log_read_stats() helper function. --- src/H5FDros3.c | 829 +++++++++++++++++++++++++------------------------ 1 file changed, 424 insertions(+), 405 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index fdff8498af1..108f240142a 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -13,10 +13,8 @@ /* * Read-Only S3 Virtual File Driver (VFD) * - * Purpose: - * - * Provide read-only access to files hosted on Amazon's S3 service. - * Relies on "s3comms" utility layer to implement the AWS REST API. + * Provides read-only access to files hosted on Amazon's S3 service. + * Relies on "s3comms" utility layer to implement the AWS REST API. */ #ifdef H5_HAVE_ROS3_VFD @@ -166,6 +164,12 @@ static int H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, static herr_t H5FD__ros3_str_token_close(const char *name, size_t size, void *_value); static herr_t H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *_value); +#ifdef ROS3_STATS +static herr_t H5FD__ros3_reset_stats(H5FD_ros3_t *file); +static herr_t H5FD__ros3_log_read_stats(H5FD_ros3_t *file, H5FD_mem_t type, uint64_t size); +static herr_t H5FD__ros3_print_stats(FILE *stream, const H5FD_ros3_t *file); +#endif + static const H5FD_class_t H5FD_ros3_g = { H5FD_CLASS_VERSION, /* struct version */ H5FD_ROS3_VALUE, /* value */ @@ -668,42 +672,6 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) FUNC_LEAVE_API(ret_value) } /* end H5Pset_fapl_ros3_token() */ -#ifdef ROS3_STATS -/*---------------------------------------------------------------------------- - * Function: H5FD__ros3_reset_stats - * - * Purpose: Reset the collected statistics - * - * Return: SUCCEED/FAIL - *---------------------------------------------------------------------------- - */ -static herr_t -H5FD__ros3_reset_stats(H5FD_ros3_t *file) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_PACKAGE - - if (file == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file was null"); - - for (int i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { - file->raw[i].bytes = 0; - file->raw[i].count = 0; - file->raw[i].min = 0; - file->raw[i].max = 0; - - file->meta[i].bytes = 0; - file->meta[i].count = 0; - file->meta[i].min = 0; - file->meta[i].max = 0; - } - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD__ros3_reset_stats() */ -#endif /* ROS3_STATS */ - /*------------------------------------------------------------------------- * Function: H5FD__ros3_open * @@ -844,371 +812,106 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__ros3_open() */ -#ifdef ROS3_STATS -/*---------------------------------------------------------------------------- - * Function: H5FD__ros3_print_stats - * - * Purpose: Tabulate and pretty-print statistics for this virtual file. - * - * Should be called upon file close. - * - * Shows number of reads and bytes read, broken down by - * "raw" (H5FD_MEM_DRAW) or "meta" (any other flag) - * - * Prints filename and listing of total number of reads and bytes read, - * both as a grand total and separate meta- and raw data reads. - * - * If any reads were done, prints out two tables: - * - * 1. overview of raw- and metadata reads - * - min (smallest size read) - * - average of size read - * - k,M,G suffixes by powers of 1024 (2^10) - * - max (largest size read) - * 2. tabulation of "bins", sepraring reads into exponentially-larger - * ranges of size. - * - columns for number of reads, total bytes, and average size, with - * separate sub-colums for raw- and metadata reads. - * - each row represents one bin, identified by the top of its range - * - * Bins without any reads in their bounds are not printed. - * - * An "overflow" bin is also present, to catch "big" reads. - * - * Output for all bins (and range ceiling and average size report) is - * divied by powers of 1024. By corollary, four digits before the decimal - * is valid. +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_close * - * - 41080 bytes is represented by 40.177k, not 41.080k - * - 1004.831M represents approx. 1052642000 bytes + * Purpose: Close the file. * * Return: SUCCEED/FAIL - *---------------------------------------------------------------------------- + *------------------------------------------------------------------------- */ static herr_t -H5FD__ros3_print_stats(FILE *stream, const H5FD_ros3_t *file) +H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file) { - herr_t ret_value = SUCCEED; - parsed_url_t *purl = NULL; - unsigned i = 0; - unsigned long count_meta = 0; - unsigned long count_raw = 0; - double average_meta = 0.0; - double average_raw = 0.0; - uint64_t min_meta = 0; - uint64_t min_raw = 0; - uint64_t max_meta = 0; - uint64_t max_raw = 0; - uint64_t bytes_raw = 0; - uint64_t bytes_meta = 0; - double re_dub = 0.0; /* reusable double variable */ - unsigned suffix_i = 0; - const char suffixes[] = {' ', 'K', 'M', 'G', 'T', 'P'}; + H5FD_ros3_t *file = (H5FD_ros3_t *)_file; + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - if (stream == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file stream cannot be null"); - if (file == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file cannot be null"); - if (file->s3r_handle == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "s3 request handle cannot be null"); - if (file->s3r_handle->purl == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "parsed url structure cannot be null"); - purl = file->s3r_handle->purl; + assert(file != NULL); + assert(file->s3r_handle != NULL); - /****************** - * PRINT FILENAME * - ******************/ +#ifdef ROS3_STATS + if (H5FD__ros3_print_stats(stdout, file) == FAIL) + HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics"); +#endif - fprintf(stream, "stats for %s://%s", purl->scheme, purl->host); - if (purl->port != NULL && purl->port[0] != '\0') - fprintf(stream, ":%s", purl->port); - if (purl->query != NULL && purl->query[0] != '\0') { - if (purl->path != NULL && purl->path[0] != '\0') - fprintf(stream, "/%s", purl->path); - else - fprintf(stream, "/"); - fprintf(stream, "?%s", purl->query); - } - else if (purl->path != NULL && purl->path[0] != '\0') { - fprintf(stream, "/%s", purl->path); - } - fprintf(stream, "\n"); + /* Close the underlying request handle */ + if (FAIL == H5FD_s3comms_s3r_close(file->s3r_handle)) + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close S3 request handle"); - /******************* - * AGGREGATE STATS * - *******************/ + /* Release the file info */ + H5MM_xfree(file->cache); + file = H5FL_FREE(H5FD_ros3_t, file); - for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { - const H5FD_ros3_stats_bin_t *r = &file->raw[i]; - const H5FD_ros3_stats_bin_t *m = &file->meta[i]; +done: + curl_global_cleanup(); - if (m->min < min_meta) - min_meta = m->min; - if (r->min < min_raw) - min_raw = r->min; - if (m->max > max_meta) - max_meta = m->max; - if (r->max > max_raw) - max_raw = r->max; + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__ros3_close() */ - count_raw += r->count; - count_meta += m->count; - bytes_raw += r->bytes; - bytes_meta += m->bytes; - } - if (count_raw > 0) - average_raw = (double)bytes_raw / (double)count_raw; - if (count_meta > 0) - average_meta = (double)bytes_meta / (double)count_meta; +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_cmp + * + * Purpose: Compares two files belonging to this driver using an + * arbitrary (but consistent) ordering: + * + * + url scheme + * + url host + * + url port + * + url path + * + url query + * + fapl aws_region + * + fapl secret_id + * + fapl secret_key + * + * TODO: This should return -1/0/1 like the other VFDs + * + * Return: Equivalent: 0 + * Not Equivalent: -1 + * (Can't fail) + *------------------------------------------------------------------------- + */ +static int +H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2) +{ + const H5FD_ros3_t *f1 = (const H5FD_ros3_t *)_f1; + const H5FD_ros3_t *f2 = (const H5FD_ros3_t *)_f2; + const parsed_url_t *purl1 = NULL; + const parsed_url_t *purl2 = NULL; + int ret_value = 0; - /****************** - * PRINT OVERVIEW * - ******************/ + FUNC_ENTER_PACKAGE_NOERR - fprintf(stream, "TOTAL READS: %lu (%lu meta, %lu raw)\n", count_raw + count_meta, count_meta, count_raw); - fprintf(stream, "TOTAL BYTES: %" PRIu64 " (%" PRIu64 " meta, %" PRIu64 " raw)\n", bytes_raw + bytes_meta, - bytes_meta, bytes_raw); + assert(f1->s3r_handle != NULL); + assert(f2->s3r_handle != NULL); - if (count_raw + count_meta == 0) - goto done; + purl1 = (const parsed_url_t *)f1->s3r_handle->purl; + purl2 = (const parsed_url_t *)f2->s3r_handle->purl; + assert(purl1 != NULL); + assert(purl2 != NULL); + assert(purl1->scheme != NULL); + assert(purl2->scheme != NULL); + assert(purl1->host != NULL); + assert(purl2->host != NULL); - /************************* - * PRINT AGGREGATE STATS * - *************************/ + /* URL: SCHEME */ + if (strcmp(purl1->scheme, purl2->scheme)) + HGOTO_DONE(-1); - fprintf(stream, "SIZES meta raw\n"); - fprintf(stream, " min "); - if (count_meta == 0) - fprintf(stream, " 0.000 "); - else { - re_dub = (double)min_meta; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) - re_dub /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - fprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); - } + /* URL: HOST */ + if (strcmp(purl1->host, purl2->host)) + HGOTO_DONE(-1); - if (count_raw == 0) - fprintf(stream, " 0.000 \n"); - else { - re_dub = (double)min_raw; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) - re_dub /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - fprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + /* URL: PORT */ + if (purl1->port && purl2->port) { + if (strcmp(purl1->port, purl2->port)) + HGOTO_DONE(-1); } - - fprintf(stream, " avg "); - re_dub = (double)average_meta; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) - re_dub /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - fprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); - - re_dub = (double)average_raw; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) - re_dub /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - fprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); - - fprintf(stream, " max "); - re_dub = (double)max_meta; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) - re_dub /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - fprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); - - re_dub = (double)max_raw; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) - re_dub /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - fprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); - - /****************************** - * PRINT INDIVIDUAL BIN STATS * - ******************************/ - - fprintf(stream, "BINS # of reads total bytes average size\n"); - fprintf(stream, " up-to meta raw meta raw meta raw\n"); - - for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { - const H5FD_ros3_stats_bin_t *m; - const H5FD_ros3_stats_bin_t *r; - uint64_t range_end = 0; - char bm_suffix = ' '; /* bytes-meta */ - double bm_val = 0.0; - char br_suffix = ' '; /* bytes-raw */ - double br_val = 0.0; - char am_suffix = ' '; /* average-meta */ - double am_val = 0.0; - char ar_suffix = ' '; /* average-raw */ - double ar_val = 0.0; - - m = &file->meta[i]; - r = &file->raw[i]; - if (r->count == 0 && m->count == 0) - continue; - - range_end = ros3_stats_boundaries_g[i]; - - if (i == ROS3_STATS_BIN_COUNT) { - range_end = ros3_stats_boundaries_g[i - 1]; - fprintf(stream, ">"); - } - else - fprintf(stream, " "); - - bm_val = (double)m->bytes; - for (suffix_i = 0; bm_val >= 1024.0; suffix_i++) - bm_val /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - bm_suffix = suffixes[suffix_i]; - - br_val = (double)r->bytes; - for (suffix_i = 0; br_val >= 1024.0; suffix_i++) - br_val /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - br_suffix = suffixes[suffix_i]; - - if (m->count > 0) - am_val = (double)(m->bytes) / (double)(m->count); - for (suffix_i = 0; am_val >= 1024.0; suffix_i++) - am_val /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - am_suffix = suffixes[suffix_i]; - - if (r->count > 0) - ar_val = (double)(r->bytes) / (double)(r->count); - for (suffix_i = 0; ar_val >= 1024.0; suffix_i++) - ar_val /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - ar_suffix = suffixes[suffix_i]; - - re_dub = (double)range_end; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) - re_dub /= 1024.0; - assert(suffix_i < sizeof(suffixes)); - - fprintf(stream, " %8.3f%c %7" PRIu64 " %7" PRIu64 " %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", re_dub, - suffixes[suffix_i], /* Bin ceiling */ - m->count, /* Metadata reads */ - r->count, /* Raw data reads */ - bm_val, bm_suffix, /* Metadata bytes */ - br_val, br_suffix, /* Raw data bytes */ - am_val, am_suffix, /* Metadata average */ - ar_val, ar_suffix); /* Raw data average */ - - fflush(stream); - } - -done: - FUNC_LEAVE_NOAPI(ret_value) - -} /* H5FD__ros3_print_stats */ -#endif /* ROS3_STATS */ - -/*------------------------------------------------------------------------- - * Function: H5FD__ros3_close - * - * Purpose: Close the file. - * - * Return: SUCCEED/FAIL - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file) -{ - H5FD_ros3_t *file = (H5FD_ros3_t *)_file; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_PACKAGE - - assert(file != NULL); - assert(file->s3r_handle != NULL); - -#ifdef ROS3_STATS - if (H5FD__ros3_print_stats(stdout, file) == FAIL) - HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics"); -#endif - - /* Close the underlying request handle */ - if (FAIL == H5FD_s3comms_s3r_close(file->s3r_handle)) - HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close S3 request handle"); - - /* Release the file info */ - H5MM_xfree(file->cache); - file = H5FL_FREE(H5FD_ros3_t, file); - -done: - curl_global_cleanup(); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD__ros3_close() */ - -/*------------------------------------------------------------------------- - * Function: H5FD__ros3_cmp - * - * Purpose: Compares two files belonging to this driver using an - * arbitrary (but consistent) ordering: - * - * + url scheme - * + url host - * + url port - * + url path - * + url query - * + fapl aws_region - * + fapl secret_id - * + fapl secret_key - * - * TODO: This should return -1/0/1 like the other VFDs - * - * Return: Equivalent: 0 - * Not Equivalent: -1 - * (Can't fail) - *------------------------------------------------------------------------- - */ -static int -H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2) -{ - const H5FD_ros3_t *f1 = (const H5FD_ros3_t *)_f1; - const H5FD_ros3_t *f2 = (const H5FD_ros3_t *)_f2; - const parsed_url_t *purl1 = NULL; - const parsed_url_t *purl2 = NULL; - int ret_value = 0; - - FUNC_ENTER_PACKAGE_NOERR - - assert(f1->s3r_handle != NULL); - assert(f2->s3r_handle != NULL); - - purl1 = (const parsed_url_t *)f1->s3r_handle->purl; - purl2 = (const parsed_url_t *)f2->s3r_handle->purl; - assert(purl1 != NULL); - assert(purl2 != NULL); - assert(purl1->scheme != NULL); - assert(purl2->scheme != NULL); - assert(purl1->host != NULL); - assert(purl2->host != NULL); - - /* URL: SCHEME */ - if (strcmp(purl1->scheme, purl2->scheme)) - HGOTO_DONE(-1); - - /* URL: HOST */ - if (strcmp(purl1->host, purl2->host)) - HGOTO_DONE(-1); - - /* URL: PORT */ - if (purl1->port && purl2->port) { - if (strcmp(purl1->port, purl2->port)) - HGOTO_DONE(-1); - } - else if (purl1->port) - HGOTO_DONE(-1); - else if (purl2->port) - HGOTO_DONE(-1); + else if (purl1->port) + HGOTO_DONE(-1); + else if (purl2->port) + HGOTO_DONE(-1); /* URL: PATH */ if (purl1->path && purl2->path) { @@ -1416,28 +1119,8 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read"); #ifdef ROS3_STATS - H5FD_ros3_stats_bin_t *bin = NULL; - int i = 0; - - /* Find which "bin" this read fits in. Can be "overflow" bin. */ - for (i = 0; i < ROS3_STATS_BIN_COUNT; i++) - if ((uint64_t)size < ros3_stats_boundaries_g[i]) - break; - bin = (type == H5FD_MEM_DRAW) ? &file->raw[i] : &file->meta[i]; - - /* Store collected stats in appropriate bin */ - if (bin->count == 0) { - bin->min = size; - bin->max = size; - } - else { - if (size < bin->min) - bin->min = size; - if (size > bin->max) - bin->max = size; - } - bin->count++; - bin->bytes += (uint64_t)size; + if (H5FD__ros3_log_read_stats(file, type, (uint64_t)size) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "unable to log read stats"); #endif } @@ -1493,4 +1176,340 @@ H5FD__ros3_truncate(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED dxpl_id, FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__ros3_truncate() */ +#ifdef ROS3_STATS +/*---------------------------------------------------------------------------- + * Function: H5FD__ros3_reset_stats + * + * Purpose: Reset the collected statistics + * + * Return: SUCCEED/FAIL + *---------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_reset_stats(H5FD_ros3_t *file) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + if (file == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file was null"); + + for (int i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { + file->raw[i].bytes = 0; + file->raw[i].count = 0; + file->raw[i].min = 0; + file->raw[i].max = 0; + + file->meta[i].bytes = 0; + file->meta[i].count = 0; + file->meta[i].min = 0; + file->meta[i].max = 0; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__ros3_reset_stats() */ + +/*---------------------------------------------------------------------------- + * Function: H5FD__ros3_log_read_stats + * + * Purpose: Add data for a read to the ros3 stats + * + * Return: SUCCEED/FAIL + *---------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_log_read_stats(H5FD_ros3_t *file, H5FD_mem_t type, uint64_t size) +{ + H5FD_ros3_stats_bin_t *bin = NULL; + int i = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + if (file == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file was null"); + + /* Find which "bin" this read fits in */ + for (i = 0; i < ROS3_STATS_BIN_COUNT; i++) + if (size < ros3_stats_boundaries_g[i]) + break; + bin = (type == H5FD_MEM_DRAW) ? &file->raw[i] : &file->meta[i]; + + /* Store collected stats in appropriate bin */ + bin->count++; + bin->bytes += size; + if (size < bin->min) + bin->min = size; + if (size > bin->max) + bin->max = size; +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__ros3_log_read_stats() */ + +/*---------------------------------------------------------------------------- + * Function: H5FD__ros3_print_stats + * + * Purpose: Tabulate and pretty-print statistics for this virtual file. + * + * Should be called upon file close. + * + * Shows number of reads and bytes read, broken down by + * "raw" (H5FD_MEM_DRAW) or "meta" (any other flag) + * + * Prints filename and listing of total number of reads and bytes read, + * both as a grand total and separate meta- and raw data reads. + * + * If any reads were done, prints out two tables: + * + * 1. overview of raw- and metadata reads + * - min (smallest size read) + * - average of size read + * - k,M,G suffixes by powers of 1024 (2^10) + * - max (largest size read) + * 2. tabulation of "bins", sepraring reads into exponentially-larger + * ranges of size. + * - columns for number of reads, total bytes, and average size, with + * separate sub-colums for raw- and metadata reads. + * - each row represents one bin, identified by the top of its range + * + * Bins without any reads in their bounds are not printed. + * + * An "overflow" bin is also present, to catch "big" reads. + * + * Output for all bins (and range ceiling and average size report) is + * divied by powers of 1024. By corollary, four digits before the decimal + * is valid. + * + * - 41080 bytes is represented by 40.177k, not 41.080k + * - 1004.831M represents approx. 1052642000 bytes + * + * Return: SUCCEED/FAIL + *---------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_print_stats(FILE *stream, const H5FD_ros3_t *file) +{ + herr_t ret_value = SUCCEED; + parsed_url_t *purl = NULL; + unsigned i = 0; + unsigned long count_meta = 0; + unsigned long count_raw = 0; + double average_meta = 0.0; + double average_raw = 0.0; + uint64_t min_meta = 0; + uint64_t min_raw = 0; + uint64_t max_meta = 0; + uint64_t max_raw = 0; + uint64_t bytes_raw = 0; + uint64_t bytes_meta = 0; + double re_dub = 0.0; /* reusable double variable */ + unsigned suffix_i = 0; + const char suffixes[] = {' ', 'K', 'M', 'G', 'T', 'P'}; + + FUNC_ENTER_PACKAGE + + if (stream == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file stream cannot be null"); + if (file == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file cannot be null"); + if (file->s3r_handle == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "s3 request handle cannot be null"); + if (file->s3r_handle->purl == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "parsed url structure cannot be null"); + purl = file->s3r_handle->purl; + + /****************** + * PRINT FILENAME * + ******************/ + + fprintf(stream, "stats for %s://%s", purl->scheme, purl->host); + if (purl->port != NULL && purl->port[0] != '\0') + fprintf(stream, ":%s", purl->port); + if (purl->query != NULL && purl->query[0] != '\0') { + if (purl->path != NULL && purl->path[0] != '\0') + fprintf(stream, "/%s", purl->path); + else + fprintf(stream, "/"); + fprintf(stream, "?%s", purl->query); + } + else if (purl->path != NULL && purl->path[0] != '\0') { + fprintf(stream, "/%s", purl->path); + } + fprintf(stream, "\n"); + + /******************* + * AGGREGATE STATS * + *******************/ + + for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { + const H5FD_ros3_stats_bin_t *r = &file->raw[i]; + const H5FD_ros3_stats_bin_t *m = &file->meta[i]; + + if (m->min < min_meta) + min_meta = m->min; + if (r->min < min_raw) + min_raw = r->min; + if (m->max > max_meta) + max_meta = m->max; + if (r->max > max_raw) + max_raw = r->max; + + count_raw += r->count; + count_meta += m->count; + bytes_raw += r->bytes; + bytes_meta += m->bytes; + } + if (count_raw > 0) + average_raw = (double)bytes_raw / (double)count_raw; + if (count_meta > 0) + average_meta = (double)bytes_meta / (double)count_meta; + + /****************** + * PRINT OVERVIEW * + ******************/ + + fprintf(stream, "TOTAL READS: %lu (%lu meta, %lu raw)\n", count_raw + count_meta, count_meta, count_raw); + fprintf(stream, "TOTAL BYTES: %" PRIu64 " (%" PRIu64 " meta, %" PRIu64 " raw)\n", bytes_raw + bytes_meta, + bytes_meta, bytes_raw); + + if (count_raw + count_meta == 0) + goto done; + + /************************* + * PRINT AGGREGATE STATS * + *************************/ + + fprintf(stream, "SIZES meta raw\n"); + fprintf(stream, " min "); + if (count_meta == 0) + fprintf(stream, " 0.000 "); + else { + re_dub = (double)min_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + fprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + } + + if (count_raw == 0) + fprintf(stream, " 0.000 \n"); + else { + re_dub = (double)min_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + fprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + } + + fprintf(stream, " avg "); + re_dub = (double)average_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + fprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)average_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + fprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + fprintf(stream, " max "); + re_dub = (double)max_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + fprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)max_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + fprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + /****************************** + * PRINT INDIVIDUAL BIN STATS * + ******************************/ + + fprintf(stream, "BINS # of reads total bytes average size\n"); + fprintf(stream, " up-to meta raw meta raw meta raw\n"); + + for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { + const H5FD_ros3_stats_bin_t *m; + const H5FD_ros3_stats_bin_t *r; + uint64_t range_end = 0; + char bm_suffix = ' '; /* bytes-meta */ + double bm_val = 0.0; + char br_suffix = ' '; /* bytes-raw */ + double br_val = 0.0; + char am_suffix = ' '; /* average-meta */ + double am_val = 0.0; + char ar_suffix = ' '; /* average-raw */ + double ar_val = 0.0; + + m = &file->meta[i]; + r = &file->raw[i]; + if (r->count == 0 && m->count == 0) + continue; + + range_end = ros3_stats_boundaries_g[i]; + + if (i == ROS3_STATS_BIN_COUNT) { + range_end = ros3_stats_boundaries_g[i - 1]; + fprintf(stream, ">"); + } + else + fprintf(stream, " "); + + bm_val = (double)m->bytes; + for (suffix_i = 0; bm_val >= 1024.0; suffix_i++) + bm_val /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + bm_suffix = suffixes[suffix_i]; + + br_val = (double)r->bytes; + for (suffix_i = 0; br_val >= 1024.0; suffix_i++) + br_val /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + br_suffix = suffixes[suffix_i]; + + if (m->count > 0) + am_val = (double)(m->bytes) / (double)(m->count); + for (suffix_i = 0; am_val >= 1024.0; suffix_i++) + am_val /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + am_suffix = suffixes[suffix_i]; + + if (r->count > 0) + ar_val = (double)(r->bytes) / (double)(r->count); + for (suffix_i = 0; ar_val >= 1024.0; suffix_i++) + ar_val /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + ar_suffix = suffixes[suffix_i]; + + re_dub = (double)range_end; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + assert(suffix_i < sizeof(suffixes)); + + fprintf(stream, " %8.3f%c %7" PRIu64 " %7" PRIu64 " %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", re_dub, + suffixes[suffix_i], /* Bin ceiling */ + m->count, /* Metadata reads */ + r->count, /* Raw data reads */ + bm_val, bm_suffix, /* Metadata bytes */ + br_val, br_suffix, /* Raw data bytes */ + am_val, am_suffix, /* Metadata average */ + ar_val, ar_suffix); /* Raw data average */ + + fflush(stream); + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD__ros3_print_stats */ +#endif /* ROS3_STATS */ + #endif /* H5_HAVE_ROS3_VFD */