Add page buffer cache command line option to tools

release_docs/RELEASE.txt

@@ -756,6 +756,14 @@ New Features
 
     Tools:
     ------
+    - Add option to adjust the page buffer size in tools
+
+      The page buffer cache size for a file can now be adjusted using the
+        --page-buffer-size=N
+      option in the h5repack, h5diff, h5dump, h5ls, and h5stat tools. This
+      will call the H5Pset_page_buffer_size() API function with the specified
+      size in bytes.
+
     - Allow h5repack to reserve space for a user block without a file
 
       This is useful for users who want to reserve space

tools/lib/h5diff.c

@@ -639,6 +639,20 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char
         }
     }
 
+    if (opts->page_cache > 0) {
+        if ((fapl1_id = h5tools_get_fapl(fapl1_id, NULL, NULL)) < 0) {
+            parallel_print("h5diff: unable to create fapl for input file\n");
+            H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n");
+        }
+
+        if (H5Pset_page_buffer_size(fapl1_id, opts->page_cache, 0, 0) < 0) {
+            error_msg("unable to set page buffer cache size for file access\n");
+            h5tools_setstatus(EXIT_FAILURE);
+            ret_value = -1;
+            goto done;
+        }
+    }
+
     if ((file1_id = h5tools_fopen(fname1, H5F_ACC_RDONLY, fapl1_id, (fapl1_id != H5P_DEFAULT), NULL,
                                   (size_t)0)) < 0) {
         parallel_print("h5diff: <%s>: unable to open file\n", fname1);
@@ -647,8 +661,13 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char
     H5TOOLS_DEBUG("file1_id = %s", fname1);
 
     /* open file 2 */
+    if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, NULL, NULL)) < 0) {
+        parallel_print("h5diff: unable to create fapl for input file\n");
+        H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n");
+    }
+
     if (opts->vfd_info[1].u.name) {
-        if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[1]))) < 0) {
+        if ((fapl2_id = h5tools_get_fapl(fapl2_id, NULL, &(opts->vfd_info[1]))) < 0) {
             parallel_print("h5diff: unable to create fapl for output file\n");
             H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n");
         }
@@ -662,6 +681,15 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char
         }
     }
 
+    if (opts->page_cache > 0) {
+        if (H5Pset_page_buffer_size(fapl2_id, opts->page_cache, 0, 0) < 0) {
+            error_msg("unable to set page buffer cache size for file access\n");
+            h5tools_setstatus(EXIT_FAILURE);
+            ret_value = -1;
+            goto done;
+        }
+    }
+
     if ((file2_id = h5tools_fopen(fname2, H5F_ACC_RDONLY, fapl2_id, (fapl2_id != H5P_DEFAULT), NULL,
                                   (size_t)0)) < 0) {
         parallel_print("h5diff: <%s>: unable to open file\n", fname2);

tools/lib/h5diff.h

@@ -92,6 +92,7 @@ typedef struct {
     h5tools_vfd_info_t        vfd_info[2];             /* VFD information for input file, output file */
     bool                      custom_vol[2];           /* Using a custom input, output VOL? */
     bool                      custom_vfd[2];           /* Using a custom input, output VFD? */
+    size_t                    page_cache;              /* Size to request for page buffer cache */
 } diff_opt_t;
 
 /*-------------------------------------------------------------------------

tools/src/h5diff/h5diff_common.c

@@ -24,7 +24,7 @@ static int check_d_input(const char *);
  * Command-line options: The user can specify short or long-named
  * parameters.
  */
-static const char            *s_opts   = "cd:ehln:p:qrv*xA:CE:NS*V";
+static const char            *s_opts   = "cd:ehln:p:qrv*xA:CE:K:NS*V";
 static struct h5_long_options l_opts[] = {{"compare", no_arg, 'c'},
                                           {"delta", require_arg, 'd'},
                                           {"use-system-epsilon", no_arg, 'e'},
@@ -39,6 +39,7 @@ static struct h5_long_options l_opts[] = {{"compare", no_arg, 'c'},
                                           {"exclude-attribute", require_arg, 'A'},
                                           {"no-compact-subset", no_arg, 'C'},
                                           {"exclude-path", require_arg, 'E'},
+                                          {"page-buffer-size", require_arg, 'K'},
                                           {"nan", no_arg, 'N'},
                                           {"enable-error-stack", optional_arg, 'S'},
                                           {"version", no_arg, 'V'},
@@ -137,6 +138,9 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const
     /**this is bad in mixing option with results**/
     opts->not_cmp = 0;
 
+    /* init for page buffer cache size option */
+    opts->page_cache = 0;
+
     /* init for exclude-path option */
     exclude_head = NULL;
 
@@ -320,6 +324,10 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const
                 opts->use_system_epsilon = 1;
                 break;
 
+            case 'K':
+                opts->page_cache = strtoul(H5_optarg, NULL, 0);
+                break;
+
             case '1':
                 opts->vol_info[0].type    = VOL_BY_VALUE;
                 opts->vol_info[0].u.value = (H5VL_class_value_t)atoi(H5_optarg);
@@ -622,6 +630,8 @@ usage(void)
     PRINTVALSTREAM(rawoutstream, "          3 : All level 2 information plus file names.\n");
     PRINTVALSTREAM(rawoutstream, "   -q, --quiet\n");
     PRINTVALSTREAM(rawoutstream, "         Quiet mode. Do not produce output.\n");
+    PRINTVALSTREAM(rawoutstream,
+                   "   --page-buffer-size=N    Set the page buffer cache size, N=non-negative integers\n");
     PRINTVALSTREAM(rawoutstream,
                    "   --vol-value-1           Value (ID) of the VOL connector to use for opening the\n");
     PRINTVALSTREAM(rawoutstream, "                           first HDF5 file specified\n");

tools/src/h5dump/h5dump.c

@@ -17,6 +17,7 @@
 /* Name of tool */
 #define PROGRAMNAME "h5dump"
 
+size_t             page_cache    = 0;
 const char        *outfname_g    = NULL;
 static bool        doxml_g       = false;
 static bool        useschema_g   = true;
@@ -97,7 +98,7 @@ struct handler_t {
  */
 /* The following initialization makes use of C language concatenating */
 /* "xxx" "yyy" into "xxxyyy". */
-static const char            *s_opts   = "a:b*c:d:ef:g:hik:l:m:n*o*pq:rs:t:uvw:xyz:A*BCD:E*F:G:HM:N:O*RS:VX:";
+static const char            *s_opts = "a:b*c:d:ef:g:hik:l:m:n*o*pq:rs:t:uvw:xyz:A*BCD:E*F:G:HK:M:N:O*RS:VX:";
 static struct h5_long_options l_opts[] = {{"attribute", require_arg, 'a'},
                                           {"binary", optional_arg, 'b'},
                                           {"count", require_arg, 'c'},
@@ -132,6 +133,7 @@ static struct h5_long_options l_opts[] = {{"attribute", require_arg, 'a'},
                                           {"form", require_arg, 'F'},
                                           {"vds-gap-size", require_arg, 'G'},
                                           {"header", no_arg, 'H'},
+                                          {"page-buffer-size", require_arg, 'K'},
                                           {"packed-bits", require_arg, 'M'},
                                           {"any_path", require_arg, 'N'},
                                           {"ddl", optional_arg, 'O'},
@@ -198,6 +200,8 @@ usage(const char *prog)
     PRINTVALSTREAM(rawoutstream, "     -O F, --ddl=F        Output ddl text into file F\n");
     PRINTVALSTREAM(rawoutstream,
                    "                          Use blank(empty) filename F to suppress ddl display\n");
+    PRINTVALSTREAM(rawoutstream,
+                   "     --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n");
     PRINTVALSTREAM(rawoutstream,
                    "     --s3-cred=<cred>     Supply S3 authentication information to \"ros3\" vfd.\n");
     PRINTVALSTREAM(rawoutstream,
@@ -1019,6 +1023,9 @@ parse_command_line(int argc, const char *const *argv)
                     goto error;
                 }
                 break;
+            case 'K':
+                page_cache = strtoul(H5_optarg, NULL, 0);
+                break;
 
             /** begin XML parameters **/
             case 'x':
@@ -1368,10 +1375,15 @@ main(int argc, char *argv[])
     /* Initialize indexing options */
     h5trav_set_index(sort_by, sort_order);
 
-    if (use_custom_vol_g || use_custom_vfd_g) {
-        if ((fapl_id = h5tools_get_fapl(H5P_DEFAULT, use_custom_vol_g ? &vol_info_g : NULL,
-                                        use_custom_vfd_g ? &vfd_info_g : NULL)) < 0) {
-            error_msg("unable to create FAPL for file access\n");
+    if ((fapl_id = h5tools_get_fapl(H5P_DEFAULT, use_custom_vol_g ? &vol_info_g : NULL,
+                                    use_custom_vfd_g ? &vfd_info_g : NULL)) < 0) {
+        error_msg("unable to create FAPL for file access\n");
+        h5tools_setstatus(EXIT_FAILURE);
+        goto done;
+    }
+    if (page_cache > 0) {
+        if (H5Pset_page_buffer_size(fapl_id, page_cache, 0, 0) < 0) {
+            error_msg("unable to set page buffer cache size for file access\n");
             h5tools_setstatus(EXIT_FAILURE);
             goto done;
         }

tools/src/h5ls/h5ls.c

@@ -25,6 +25,7 @@
 #define PROGRAMNAME "h5ls"
 
 #define NAME_BUF_SIZE 2048
+
 /*
  *  Alternative formatting for data dumped by H5LS
  *
@@ -211,6 +212,8 @@ usage(void)
     PRINTVALSTREAM(rawoutstream, "   -wN, --width=N  Set the number of columns of output\n");
     PRINTVALSTREAM(rawoutstream, "   -v, --verbose   Generate more verbose output\n");
     PRINTVALSTREAM(rawoutstream, "   -V, --version   Print version number and exit\n");
+    PRINTVALSTREAM(rawoutstream,
+                   "   --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n");
     PRINTVALSTREAM(rawoutstream, "   --vfd=DRIVER    Use the specified virtual file driver\n");
     PRINTVALSTREAM(rawoutstream, "   -x, --hexdump   Show raw data in hexadecimal format\n");
     PRINTVALSTREAM(rawoutstream,
@@ -2659,6 +2662,7 @@ main(int argc, char *argv[])
     static char        root_name[] = "/";
     char               drivername[50];
     int                err_exit        = 0;
+    size_t             page_cache      = 0;
     hid_t              fapl_id         = H5P_DEFAULT;
     bool               custom_vol_fapl = false;
     bool               custom_vfd_fapl = false;
@@ -2816,6 +2820,9 @@ main(int argc, char *argv[])
                 leave(EXIT_FAILURE);
             }
         }
+        else if (!strncmp(argv[argno], "--page-buffer-size=", (size_t)19)) {
+            page_cache = strtoul(argv[argno] + 8, &rest, 0);
+        }
         else if (!strcmp(argv[argno], "--verbose")) {
             verbose_g++;
         }
@@ -2995,23 +3002,27 @@ main(int argc, char *argv[])
     }
 
     /* Setup a custom fapl for file accesses */
-    if (custom_vol_fapl || custom_vfd_fapl) {
 #ifdef H5_HAVE_ROS3_VFD
-        if (custom_vfd_fapl && (0 == strcmp(vfd_info.u.name, drivernames[ROS3_VFD_IDX]))) {
-            if (!vfd_info.info)
-                vfd_info.info = &ros3_fa;
-        }
+    if (custom_vfd_fapl && (0 == strcmp(vfd_info.u.name, drivernames[ROS3_VFD_IDX]))) {
+        if (!vfd_info.info)
+            vfd_info.info = &ros3_fa;
+    }
 #endif
 #ifdef H5_HAVE_LIBHDFS
-        if (custom_vfd_fapl && (0 == strcmp(vfd_info.u.name, drivernames[HDFS_VFD_IDX]))) {
-            if (!vfd_info.info)
-                vfd_info.info = &hdfs_fa;
-        }
+    if (custom_vfd_fapl && (0 == strcmp(vfd_info.u.name, drivernames[HDFS_VFD_IDX]))) {
+        if (!vfd_info.info)
+            vfd_info.info = &hdfs_fa;
+    }
 #endif
 
-        if ((fapl_id = h5tools_get_fapl(H5P_DEFAULT, custom_vol_fapl ? &vol_info : NULL,
-                                        custom_vfd_fapl ? &vfd_info : NULL)) < 0) {
-            error_msg("failed to setup file access property list (fapl) for file\n");
+    if ((fapl_id = h5tools_get_fapl(H5P_DEFAULT, custom_vol_fapl ? &vol_info : NULL,
+                                    custom_vfd_fapl ? &vfd_info : NULL)) < 0) {
+        error_msg("failed to setup file access property list (fapl) for file\n");
+        leave(EXIT_FAILURE);
+    }
+    if (page_cache > 0) {
+        if (H5Pset_page_buffer_size(fapl_id, page_cache, 0, 0) < 0) {
+            error_msg("unable to set page buffer cache size for file access\n");
             leave(EXIT_FAILURE);
         }
     }

tools/src/h5repack/h5repack_main.c

@@ -30,7 +30,8 @@ static const char *outfile = NULL;
  * Command-line options: The user can specify short or long-named
  * parameters.
  */
-static const char *s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:v*z:E*G:LM:P:S:T:VXWY:Z:1:2:3:4:5:6:7:8:9:0:";
+static const char *s_opts =
+    "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:v*z:E*G:K:LM:P:S:T:VXWY:Z:1:2:3:4:5:6:7:8:9:0:";
 static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'},
                                           {"block", require_arg, 'b'},
                                           {"compact", require_arg, 'c'},
@@ -53,6 +54,7 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'},
                                           {"sort_order", require_arg, 'z'},
                                           {"enable-error-stack", optional_arg, 'E'},
                                           {"fs_pagesize", require_arg, 'G'},
+                                          {"page-buffer-size", require_arg, 'K'},
                                           {"latest", no_arg, 'L'},
                                           {"metadata_block_size", require_arg, 'M'},
                                           {"fs_persist", require_arg, 'P'},
@@ -113,6 +115,8 @@ usage(const char *prog)
     PRINTVALSTREAM(rawoutstream, "      N - is an integer greater than 1, 2 displays read/write timing\n");
     PRINTVALSTREAM(rawoutstream, "   -V, --version           Print version number and exit\n");
     PRINTVALSTREAM(rawoutstream, "   -n, --native            Use a native HDF5 type when repacking\n");
+    PRINTVALSTREAM(rawoutstream,
+                   "   --page-buffer-size=N    Set the page buffer cache size, N=non-negative integers\n");
     PRINTVALSTREAM(rawoutstream,
                    "   --src-vol-value         Value (ID) of the VOL connector to use for opening the\n");
     PRINTVALSTREAM(rawoutstream, "                           input HDF5 file specified\n");
@@ -531,6 +535,7 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options)
     bool               custom_out_vol = false;
     bool               custom_out_vfd = false;
     hid_t              tmp_fapl       = H5I_INVALID_HID;
+    size_t             page_cache     = 0;
     int                bound, opt;
     int                ret_value = 0;
 
@@ -765,6 +770,10 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options)
                     options->fs_pagesize = -1;
                 break;
 
+            case 'K':
+                page_cache = strtoul(H5_optarg, NULL, 0);
+                break;
+
             case 'q':
                 if (H5_INDEX_UNKNOWN == (sort_by = set_sort_by(H5_optarg))) {
                     error_msg("failed to set sort by form <%s>\n", H5_optarg);
@@ -924,6 +933,32 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options)
 
         options->fin_fapl = tmp_fapl;
     }
+    if (page_cache > 0) {
+        if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, NULL, NULL)) < 0) {
+            error_msg("failed to setup FAPL for input file\n");
+            h5tools_setstatus(EXIT_FAILURE);
+            ret_value = -1;
+            goto done;
+        }
+
+        if (H5Pset_page_buffer_size(tmp_fapl, page_cache, 0, 0) < 0) {
+            error_msg("unable to set page buffer cache size for file access\n");
+            h5tools_setstatus(EXIT_FAILURE);
+            ret_value = -1;
+            goto done;
+        }
+
+        /* Close old FAPL */
+        if (options->fin_fapl != H5P_DEFAULT)
+            if (H5Pclose(options->fin_fapl) < 0) {
+                error_msg("failed to close FAPL\n");
+                h5tools_setstatus(EXIT_FAILURE);
+                ret_value = -1;
+                goto done;
+            }
+
+        options->fin_fapl = tmp_fapl;
+    }
 
     if (custom_out_vol || custom_out_vfd) {
         if ((tmp_fapl = h5tools_get_fapl(options->fout_fapl, custom_out_vol ? &out_vol_info : NULL,