From 6fe9dc62d810f5e0b4bbc5616168c2c5750094a7 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Mon, 15 Nov 2021 14:31:17 -0600 Subject: [PATCH 01/39] Add fbh5_context_t, fbh5_ctx_ics and fbh5_ctx_ant definitions --- rawspec_callback.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/rawspec_callback.h b/rawspec_callback.h index 6609b3e..ad6abca 100644 --- a/rawspec_callback.h +++ b/rawspec_callback.h @@ -4,6 +4,20 @@ #include #include "rawspec_fbutils.h" +typedef struct { + hid_t file_id; // File-level handle (similar to an fd) + hid_t dataset_id; // Dataset "data" handle + hid_t dataspace_id; // Dataspace handle for dataset "data" + unsigned int elem_size; // Byte size of one spectra element (E.g. 4 if nbits=32) + hid_t elem_type; // HDF5 type for all elements (derived from nbits in fbh5_open) + size_t tint_size; // Size of a time integration (computed in fbh5_open) + hsize_t offset_dims[3]; // Next offset dimensions for the fbh5_write function + // (offset_dims[0] : time integration count) + hsize_t filesz_dims[3]; // Accumulated file size in dimensions + unsigned long byte_count; // Number of bytes output so far + unsigned long dump_count; // Number of dumps processed so far +} fbh5_context_t; + typedef struct { int *fd; // Output file descriptors (one for each antenna) or socket (at most 1) int fd_ics; // Output file descriptor or socket @@ -27,6 +41,11 @@ typedef struct { unsigned int Nf; // Number of fine channels (== Nc*Nts[i]) // Filterbank header fb_hdr_t fb_hdr; + + // Added for FBH5 2021-11-15 + fbh5_context_t fbh5_ctx_ics; // Singleton fbh5 ctx for ics + fbh5_context_t * fbh5_ctx_ant; // Pointer to array of fbh5 ctx for individual antennas + } callback_data_t; #endif // _RAWSPEC_CALLBACK_H_ From 45fe705d9c506b8ae99ea20c170b9223d36c7e1f Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Mon, 15 Nov 2021 14:35:08 -0600 Subject: [PATCH 02/39] Add #include --- rawspec_callback.h | 1 + 1 file changed, 1 insertion(+) diff --git a/rawspec_callback.h b/rawspec_callback.h index ad6abca..d744e30 100644 --- a/rawspec_callback.h +++ b/rawspec_callback.h @@ -2,6 +2,7 @@ #define _RAWSPEC_CALLBACK_H_ #include +#include #include "rawspec_fbutils.h" typedef struct { From b4cf4b176b786af4adb7a6130213ad4047103fe9 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Mon, 15 Nov 2021 14:40:03 -0600 Subject: [PATCH 03/39] Added nfpc to the fb_hdr structure --- rawspec_fbutils.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rawspec_fbutils.h b/rawspec_fbutils.h index a956d13..5f9150e 100644 --- a/rawspec_fbutils.h +++ b/rawspec_fbutils.h @@ -108,6 +108,12 @@ typedef struct { char source_name[81]; // the name of the original data file char rawdatafile[81]; + + // ------------------------ new for FBH5 ------------------------------------ + // Number of fine channels per coarse channel + // E.g. turbo_seti will compute the number of coarse channels = nchans / nfpc + int nfpc; + } fb_hdr_t; #ifdef __cplusplus From 5d21857be82fbd4dec8ccaf52c3a78c8ca4a04e3 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Mon, 15 Nov 2021 14:45:08 -0600 Subject: [PATCH 04/39] Added fbh5*.c and fbh5_defs.h --- fbh5_close.c | 66 ++++++++++++++++ fbh5_defs.h | 62 +++++++++++++++ fbh5_open.c | 193 +++++++++++++++++++++++++++++++++++++++++++++++ fbh5_util.c | 207 +++++++++++++++++++++++++++++++++++++++++++++++++++ fbh5_write.c | 134 +++++++++++++++++++++++++++++++++ 5 files changed, 662 insertions(+) create mode 100644 fbh5_close.c create mode 100644 fbh5_defs.h create mode 100644 fbh5_open.c create mode 100644 fbh5_util.c create mode 100644 fbh5_write.c diff --git a/fbh5_close.c b/fbh5_close.c new file mode 100644 index 0000000..70091e8 --- /dev/null +++ b/fbh5_close.c @@ -0,0 +1,66 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * fbh5_close.c * + * ---------- * + * Close an FBH5 writing session: . * + * Dataspace, Dataset, and File (in that order). * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include "fbh5_defs.h" +#define MILLION 1000000.0 + + +/*** + Main entry point. +***/ +void fbh5_close(fbh5_context_t * p_fbh5_ctx, int debug_callback) { + herr_t status; // Status from HDF5 function call + hsize_t sz_store; // Storage size + double MiBstore; // sz_store converted to MiB + double MiBlogical; // sz_store converted to MiB + + sz_store = H5Dget_storage_size(p_fbh5_ctx->dataset_id); + MiBlogical = (double) p_fbh5_ctx->tint_size * (double) p_fbh5_ctx->offset_dims[0] / MILLION; + + /* + * Attach "dimension scale" labels. + */ + fbh5_set_ds_label(p_fbh5_ctx, "time", 0, debug_callback); + fbh5_set_ds_label(p_fbh5_ctx, "feed_id", 1, debug_callback); + fbh5_set_ds_label(p_fbh5_ctx, "frequency", 2, debug_callback); + + /* + * Close dataspace. + */ + status = H5Sclose(p_fbh5_ctx->dataspace_id); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_close H5Sclose dataspace FAILED\n"); + + /* + * Close dataset. + */ + status = H5Dclose(p_fbh5_ctx->dataset_id); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_close H5Dclose dataset 'data' FAILED\n"); + + /* + * Close file. + */ + status = H5Fclose(p_fbh5_ctx->file_id); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_close H5Fclose FAILED\n"); + + /* + * Closing statistics. + */ + printf("fbh5_close: Context closed.\n"); + printf("fbh5_close: %ld dumps processed.\n", p_fbh5_ctx->dump_count); + printf("fbh5_close: %lld time integrations processed.\n", p_fbh5_ctx->offset_dims[0]); + MiBstore = (double) sz_store / MILLION; + printf("fbh5_close: Compressed %.2f MiB --> %.2f MiB\n", MiBlogical, MiBstore); + + /* + * Bye-bye. + */ +} + diff --git a/fbh5_defs.h b/fbh5_defs.h new file mode 100644 index 0000000..e0a068e --- /dev/null +++ b/fbh5_defs.h @@ -0,0 +1,62 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * fbh5_defs.h * + * ----------- * + * Global Definitions . * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef FBH5_DEFS_H +#define FBH5_DEFS_H + +#include +#include +#include +#include +#include +#include + +/* + * HDF5 library definitions + */ +#include +#include +#include +#ifndef H5_HAVE_THREADSAFE +#error The installed HDF5 run-time is not thread-safe! +#endif + +/* + * Rawspec callback definitions + */ +#include "rawspec_callback.h" + +/* + * Global definitions + */ +#define DATASETNAME "data" // FBH5 dataset name to hold the data matrix +#define NDIMS 3 // # of data matrix dimensions (rank) +#define FILTERBANK_CLASS "FILTERBANK" // File-level attribute "CLASS" +#define FILTERBANK_VERSION "2.0" // File-level attribute "VERSION" + +/* + * fbh5 API functions + */ +void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_path, int debug_callback); +void fbh5_write(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, void * buffer, size_t bufsize, int debug_callback); +void fbh5_close(fbh5_context_t * p_fbh5_ctx, int debug_callback); + +/* + * fbh5_util.c functions + */ +void fbh5_oops(char * srcfile, int linenum, char * msg); +void fbh5_set_str_attr(hid_t file_or_dataset_id, char * tag, char * value, int debug_callback); +void fbh5_set_dataset_double_attr(hid_t dataset_id, char * tag, double * p_value, int debug_callback); +void fbh5_set_dataset_int_attr(hid_t dataset_id, char * tag, int * p_value, int debug_callback); +void fbh5_write_metadata(hid_t dataset_id, fb_hdr_t * p_metadata, int debug_callback); +void fbh5_set_ds_label(fbh5_context_t * p_fbh5_ctx, char * label, int dims_index, int debug_callback); + +/* + * Debugging aid + */ +void hexdump(const char *label, const void *thingaddr, const int thingsz, int per_line); + +#endif diff --git a/fbh5_open.c b/fbh5_open.c new file mode 100644 index 0000000..9a2783f --- /dev/null +++ b/fbh5_open.c @@ -0,0 +1,193 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * fbh5_open.c * + * ----------- * + * Begin an FBH5 writing session . * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include "fbh5_defs.h" + + +unsigned COMPRESSION = 1; +unsigned SHUFFLE = 1; + + +/*** + Main entry point. +***/ +void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_path, int debug_callback) { + hid_t dcpl; // Chunking handle - needed until dataset handle is produced + hsize_t max_dims[NDIMS]; // Maximum dataset allocation dimensions + hsize_t cdims[NDIMS]; // Chunking dimensions + herr_t status; // Status from HDF5 function call + char wstr[256]; // sprintf target + unsigned hdf5_majnum, hdf5_minnum, hdf5_relnum; // Version/release info for the HDF5 library + + int deflate_level = 6; // 0=no deflation, 9=highest, 6=recommended + + H5get_libversion(&hdf5_majnum, &hdf5_minnum, &hdf5_relnum); + printf("fbh5_open: FBH5 path: %s\n", output_path); + printf("fbh5_open: HDF5 library version: %d.%d.%d\n", hdf5_majnum, hdf5_minnum, hdf5_relnum); + printf("fbh5_open: Creating dataspace dimensions using nifs=%d and nchans=%d\n", + p_fb_hdr->nifs, + p_fb_hdr->nchans); + + /* + * Validate fb_hdr: nifs, nbits, nfpc, nchans. + */ + if((p_fb_hdr->nbits % 8 != 0) || (p_fb_hdr->nbits < 8) || (p_fb_hdr->nbits > 64)) { + sprintf(wstr, "fbh5_open: nbits must be in [8, 16, 32, 64] but I saw %d", p_fb_hdr->nifs); + fbh5_oops(__FILE__, __LINE__, wstr); + } + if((p_fb_hdr->nifs < 1) || (p_fb_hdr->nifs > 4)) { + sprintf(wstr, "fbh5_open: nifs must be in [1, 2, 3, 4] but I saw %d", p_fb_hdr->nifs); + fbh5_oops(__FILE__, __LINE__, wstr); + } + if(p_fb_hdr->nfpc < 1) { + sprintf(wstr, "fbh5_open: nfpc must be > 0 but I saw %d", p_fb_hdr->nfpc); + fbh5_oops(__FILE__, __LINE__, wstr); + } + if(p_fb_hdr->nchans < 1) { + sprintf(wstr, "fbh5_open: nchans must be > 0 but I saw %d", p_fb_hdr->nchans); + fbh5_oops(__FILE__, __LINE__, wstr); + } + + /* + * Initialize FBH5 context. + */ + memset(p_fbh5_ctx, 0, sizeof(fbh5_context_t)); + p_fbh5_ctx->elem_size = p_fb_hdr->nbits / 8; + p_fbh5_ctx->tint_size = p_fb_hdr->nifs * p_fb_hdr->nchans * p_fbh5_ctx->elem_size; + p_fbh5_ctx->offset_dims[0] = 0; + p_fbh5_ctx->offset_dims[1] = 0; + p_fbh5_ctx->offset_dims[2] = 0; + printf("fbh5_open: Data element byte size = %d.\n", p_fbh5_ctx->elem_size); + printf("fbh5_open: Time integration byte size = %ld.\n", p_fbh5_ctx->tint_size); + + /* + * Open HDF5 file. Overwrite it if preexisting. + */ + p_fbh5_ctx->file_id = H5Fcreate(output_path, // Full path of output file + H5F_ACC_TRUNC, // Overwrite if preexisting. + H5P_DEFAULT, // Default creation property list + H5P_DEFAULT); // Default access property list + if(p_fbh5_ctx->file_id < 0) { + sprintf(wstr, "fbh5_open: H5Fcreate of '%s' FAILED", output_path); + fbh5_oops(__FILE__, __LINE__, wstr); + } + + /* + * Write the file-level metadata attributes. + */ + fbh5_set_str_attr(p_fbh5_ctx->file_id, "CLASS", FILTERBANK_CLASS, debug_callback); + fbh5_set_str_attr(p_fbh5_ctx->file_id, "VERSION", FILTERBANK_VERSION, debug_callback); + + /* + * Initialise the total file size in terms of its shape. + */ + p_fbh5_ctx->filesz_dims[0] = 1; + p_fbh5_ctx->filesz_dims[1] = p_fb_hdr->nifs; + p_fbh5_ctx->filesz_dims[2] = p_fb_hdr->nchans; + + /* + * Set the maximum file size in terms of its shape. + */ + max_dims[0] = H5S_UNLIMITED; + max_dims[1] = p_fb_hdr->nifs; + max_dims[2] = p_fb_hdr->nchans; + + /* + * Create a dataspace which is extensible in the time dimension. + */ + p_fbh5_ctx->dataspace_id = H5Screate_simple(NDIMS, // Rank + p_fbh5_ctx->filesz_dims, // initial dimensions + max_dims); // maximum dimensions + if(p_fbh5_ctx->dataspace_id < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Screate_simple FAILED"); + + /* + * Initialise the dataset creation property list + */ + dcpl = H5Pcreate(H5P_DATASET_CREATE); + if(dcpl < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pcreate/dcpl FAILED"); + + /* + * Add deflating to the dataset creation property list. + */ + if(COMPRESSION) { + if(SHUFFLE) { + status = H5Pset_shuffle(dcpl); + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_shuffle FAILED"); + printf("fbh5_open: Using shuffle deflation.\n"); + } + else + printf("fbh5_open: Using gzip deflation.\n"); + status = H5Pset_deflate(dcpl, deflate_level); + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_deflate FAILED"); + } + + /* + * Add chunking to the dataset creation property list. + */ + cdims[0] = 1; + cdims[1] = 1; + cdims[2] = p_fb_hdr->nchans; + status = H5Pset_chunk(dcpl, NDIMS, cdims); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_chunk FAILED"); + printf("fbh5_open: Chunking (%lld, %lld, %lld) configured\n", cdims[0], cdims[1], cdims[2]); + + /* + * Define datatype for the data in the file. + * We will store little endian values. + */ + switch(p_fb_hdr->nbits) { + case 8: + p_fbh5_ctx->elem_type = H5T_NATIVE_B8; + break; + case 16: + p_fbh5_ctx->elem_type = H5T_NATIVE_B16; + break; + case 32: + p_fbh5_ctx->elem_type = H5T_IEEE_F32LE; + break; + default: // 64 + p_fbh5_ctx->elem_type = H5T_IEEE_F64LE; + } + + /* + * Create the dataset. + */ + p_fbh5_ctx->dataset_id = H5Dcreate(p_fbh5_ctx->file_id, // File handle + DATASETNAME, // Dataset name + p_fbh5_ctx->elem_type, // HDF5 data type + p_fbh5_ctx->dataspace_id, // Dataspace handle + H5P_DEFAULT, // + dcpl, // Dataset creation property list + H5P_DEFAULT); // Default access properties + if(p_fbh5_ctx->dataset_id < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Dcreate FAILED"); + + /* + * Close dcpl handle. + */ + status = H5Pclose(dcpl); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pclose/dcpl FAILED\n"); + + /* + * Write dataset metadata attributes. + */ + fbh5_write_metadata(p_fbh5_ctx->dataset_id, // Dataset handle + p_fb_hdr, // Metadata (SIGPROC header) + debug_callback); // Tracing flag + printf("fbh5_open: Dataset metadata stored; done.\n"); + + /* + * Bye-bye. + */ +} + diff --git a/fbh5_util.c b/fbh5_util.c new file mode 100644 index 0000000..7ac16fe --- /dev/null +++ b/fbh5_util.c @@ -0,0 +1,207 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * fbh5_util.c * + * ----------- * + * Utility functions . * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include "fbh5_defs.h" + + +/*** + Report bad news, cleanup, and exit to OS +***/ +void fbh5_oops(char * srcfile, int linenum, char * msg) { + fprintf(stderr, "\n*** OOPS (fbh5), fatal error detected in %s at line %d !!\n*** %s !!\n\n", srcfile, linenum, msg); + exit(86); +} + + +/*** + Set a file-level or dataset-level attribute in string-value format. +***/ +void fbh5_set_str_attr(hid_t file_or_dataset_id, char * tag, char * value, int debug_callback) { + herr_t status; + hid_t id_scalar, atype, id_attr; + + if(debug_callback) + printf("fbh5_set_str_attr: %s = %s\n", tag, value); + id_scalar = H5Screate(H5S_SCALAR); + atype = H5Tcopy(H5T_C_S1); // attr type = string + H5Tset_size(atype, strlen(value)); // set create the string + H5Tset_strpad(atype, H5T_STR_NULLTERM); + id_attr = H5Acreate(file_or_dataset_id, tag, atype, id_scalar, H5P_DEFAULT, H5P_DEFAULT); + status = H5Awrite(id_attr, atype, value); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_set_str_attr H5Awrite set_file_str_attr FAILED"); + status = H5Aclose(id_attr); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_set_str_attr H5Aclose set_file_str_attr FAILED"); +} + + +/*** + Set a dataset-level attribute in double-value format. +***/ +void fbh5_set_dataset_double_attr(hid_t dataset_id, char * tag, double * p_value, int debug_callback) { + herr_t status; + hid_t id_attr, id_scalar; + + if(debug_callback) + printf("fbh5_set_dataset_double_attr: %s = %f\n", tag, *p_value); + id_scalar = H5Screate(H5S_SCALAR); + if(id_scalar < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_set_dataset_double_attr H5Screate FAILED"); + id_attr = H5Acreate2(dataset_id, tag, H5T_NATIVE_DOUBLE, id_scalar, H5P_DEFAULT, H5P_DEFAULT); + if(id_attr < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_set_dataset_double_attr H5Acreate2 FAILED"); + status = H5Awrite(id_attr, H5T_NATIVE_DOUBLE, p_value); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_set_dataset_double_attr H5Awrite FAILED"); + status = H5Aclose(id_attr); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_set_dataset_double_attr H5Aclose FAILED"); +} + + +/*** + Set a dataset-level attribute in int-value format. +***/ +void fbh5_set_dataset_int_attr(hid_t dataset_id, char * tag, int * p_value, int debug_callback) { + herr_t status; + hid_t id_attr, id_scalar; + + if(debug_callback) + printf("fbh5_set_dataset_int_attr: %s = %d\n", tag, *p_value); + id_scalar = H5Screate(H5S_SCALAR); + if(id_scalar < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_set_dataset_int_attr H5Screate FAILED"); + id_attr = H5Acreate2(dataset_id, tag, H5T_NATIVE_INT, id_scalar, H5P_DEFAULT, H5P_DEFAULT); + if(id_attr < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_set_dataset_int_attr H5Acreate2 FAILED"); + status = H5Awrite(id_attr, H5T_NATIVE_INT, p_value); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_set_dataset_int_attr H5Awrite FAILED"); + status = H5Aclose(id_attr); + if(status != 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_set_dataset_int_attr H5Aclose FAILED"); +} + + +/*** + Write metadata to FBH5 file dataset. +***/ +void fbh5_write_metadata(hid_t dataset_id, fb_hdr_t *p_md, int debug_callback) { + fbh5_set_dataset_int_attr(dataset_id, "machine_id", &(p_md->machine_id), debug_callback); + fbh5_set_dataset_int_attr(dataset_id, "telescope_id", &(p_md->telescope_id), debug_callback); + fbh5_set_dataset_int_attr(dataset_id, "data_type", &(p_md->data_type), debug_callback); + fbh5_set_dataset_int_attr(dataset_id, "nchans", &(p_md->nchans), debug_callback); + fbh5_set_dataset_int_attr(dataset_id, "nfpc", &(p_md->nfpc), debug_callback); + fbh5_set_dataset_int_attr(dataset_id, "nbeams", &(p_md->nbeams), debug_callback); + fbh5_set_dataset_int_attr(dataset_id, "ibeam", &(p_md->ibeam), debug_callback); + fbh5_set_dataset_int_attr(dataset_id, "nbits", &(p_md->nbits), debug_callback); + fbh5_set_dataset_int_attr(dataset_id, "nifs", &(p_md->nifs), debug_callback); + fbh5_set_dataset_double_attr(dataset_id, "src_raj", &(p_md->src_raj), debug_callback); + fbh5_set_dataset_double_attr(dataset_id, "src_dej", &(p_md->src_dej), debug_callback); + fbh5_set_dataset_double_attr(dataset_id, "az_start", &(p_md->az_start), debug_callback); + fbh5_set_dataset_double_attr(dataset_id, "za_start", &(p_md->za_start), debug_callback); + fbh5_set_dataset_double_attr(dataset_id, "fch1", &(p_md->fch1), debug_callback); + fbh5_set_dataset_double_attr(dataset_id, "foff", &(p_md->foff), debug_callback); + fbh5_set_dataset_double_attr(dataset_id, "tstart", &(p_md->tstart), debug_callback); + fbh5_set_dataset_double_attr(dataset_id, "tsamp", &(p_md->tsamp), debug_callback); + fbh5_set_str_attr(dataset_id, "source_name", &(p_md->source_name[0]), debug_callback); + fbh5_set_str_attr(dataset_id, "rawdatafile", &(p_md->rawdatafile[0]), debug_callback); +} + +/*** + Set up a dimension scale label. + + * Create a secondary dataset, dscale_id. + * Attach dscale_id to the file's primary dataset as a Dimension Scale label. +***/ +void fbh5_set_ds_label(fbh5_context_t * p_fbh5_ctx, char * label, int dims_index, int debug_callback) { + herr_t status; + char wstr[256]; + + if(debug_callback) + printf("fbh5_set_ds_label: label = %s, dims_index = %d\n", label, dims_index); + status = H5DSset_label(p_fbh5_ctx->dataset_id, // Dataset ID + dims_index, // Dimension index to which dscale_id applies to + label); // Label + if(status < 0) { + sprintf(wstr, "fbh5_set_ds_label H5DSset_label FAILED (%s)", label); + fbh5_oops(__FILE__, __LINE__, wstr); + } +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * hexdump.c * + * --------- * + * Cheap-o-cigar dump anything-to-hex-and-string on stdout * + * * + * Usage: * + * hexdump(label, addr, len, per_line); * + * * + * where: * + * label: label of thing being dumped. * + * thingaddr: address to start dumping from. * + * thingsz: number of bytes to dump. * + * per_line: number of bytes on each output line. * + * * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +void hexdump(const char *label, const void *thingaddr, const int thingsz, int per_line) { + // Silently ignore silly per-line values. + if(per_line < 4 || per_line > 64) per_line = 16; + + int ii; // index into buffer, byte for byte + unsigned char buffer[per_line+1]; + const unsigned char * pc = (const unsigned char *)thingaddr; + + // Output description if given. + if(label != NULL) + printf("hexdump %s (%d bytes):\n", label, thingsz); + else + printf("hexdump (no label) %d bytes\n", thingsz); + + // Length checks. + if(thingsz < 1) { + fprintf(stderr, "\nhexdump: thingsz(%d) < 1; label=%s\n", thingsz, label); + exit(86); + } + + // Process every byte in the data. + for (ii = 0; ii < thingsz; ii++) { + // Multiple of per_line means new or first line (with line offset). + if((ii % per_line) == 0) { + // Only print previous-line ASCII buffer for lines beyond first. + if(ii != 0) printf(" %s\n", buffer); + + // Output the offset of current line. + printf(" %04x ", ii); + } + + // Now the hex code for the specific character. + printf(" %02x", pc[ii]); + + // And buffer a printable ASCII character for later. + if((pc[ii] < 0x20) || (pc[ii] > 0x7e)) // isprint() may be better. + buffer[ii % per_line] = '.'; + else + buffer[ii % per_line] = pc[ii]; + + // Add a C string terminator. + buffer[(ii % per_line) + 1] = '\0'; + } + + // Pad out last line if not exactly per_line characters. + while ((ii % per_line) != 0) { + printf(" "); + ii++; + } + + // Dump the final ASCII buffer. + printf(" %s\n", buffer); +} + + diff --git a/fbh5_write.c b/fbh5_write.c new file mode 100644 index 0000000..a9a5620 --- /dev/null +++ b/fbh5_write.c @@ -0,0 +1,134 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * fbh5_write.c * + * ------------ * + * Write an FBH5 dump. . * + * * + * HDF 5 library functions used: * + * - H5Dset_extent - Define new file size to include this new dump * + * - H5Dget_space - Get a space handle for writing * + * - H5Sselect_hyperslab - Define hyperslab offset and length in to write * + * - H5Dwrite - Write the hyperslab * + * * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include "fbh5_defs.h" + +/*** + Main entry point. +***/ +void fbh5_write(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, void * p_buffer, size_t bufsize, int debug_callback) { + herr_t status; // Status from HDF5 function call + size_t ntints; // Number of time integrations in the current dump + hid_t filespace_id; // Identifier for a copy of the dataspace + hsize_t selection[3]; // Current selection + void *bufptr; // Pointer into the current dump + + /* + * Initialise write loop. + */ + ntints = bufsize / p_fbh5_ctx->tint_size; // Compute the number of time integrations in the current dump. + p_fbh5_ctx->dump_count += 1; // Bump the dump count. + bufptr = p_buffer; // Point to the first time integration in the dump. + + /* + * BEGIN write-loop for the current dump. + * Write one time integration at a time. + */ + for(int ii = 0; ii < ntints; ++ii) { + /* + * Bump the count of time integrations. + * If this is the very first write, no need to bump the file size + * because fbh5_open already accounted for the first time integration. + */ + if(p_fbh5_ctx->offset_dims[0] > 0) + p_fbh5_ctx->filesz_dims[0] += 1; + + /* + * Define the current slab selection in terms of its shape. + */ + selection[0] = 1; + selection[1] = p_fb_hdr->nifs; + selection[2] = p_fb_hdr->nchans; + + if(debug_callback) { + printf("fbh5_write: dump %ld, dump-tint %d, offset=(%lld, %lld, %lld), selection=(%lld, %lld, %lld), filesize=(%lld, %lld, %lld)\n", + p_fbh5_ctx->dump_count, + ii + 1, + p_fbh5_ctx->offset_dims[0], + p_fbh5_ctx->offset_dims[1], + p_fbh5_ctx->offset_dims[2], + selection[0], + selection[1], + selection[2], + p_fbh5_ctx->filesz_dims[0], + p_fbh5_ctx->filesz_dims[1], + p_fbh5_ctx->filesz_dims[2]); + } + status = H5Dset_extent(p_fbh5_ctx->dataset_id, // Dataset handle + p_fbh5_ctx->filesz_dims); // New file size shape + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Dset_extent/dataset_id FAILED"); + + /* + * Get filespace. + */ + filespace_id = H5Dget_space(p_fbh5_ctx->dataset_id); // Dataset handle + if(filespace_id < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Dget_space FAILED"); + + /* + * Select the filespace hyperslab. + */ + status = H5Sselect_hyperslab(filespace_id, // Filespace handle + H5S_SELECT_SET, // Replace preexisting selection + p_fbh5_ctx->offset_dims, // Starting offset dimensions of first element + NULL, // Not "striding" + selection, // Selection dimensions + NULL); // Block parameter : default value + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Sselect_hyperslab/filespace FAILED"); + + /* + * Write out current time integration to the hyperslab. + */ + status = H5Dwrite(p_fbh5_ctx->dataset_id, // Dataset handle + p_fbh5_ctx->elem_type, // HDF5 element type + p_fbh5_ctx->dataspace_id, // Dataspace handle + filespace_id, // Filespace_id + H5P_DEFAULT, // Default data transfer properties + bufptr); // Buffer holding the data + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Dwrite FAILED"); + + /* + * Close temporary filespace handle. + */ + status = H5Sclose(filespace_id); + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_close H5Sclose/filespace_id FAILED\n"); + /* + * Point ahead for the next call to fbh5_write. + */ + p_fbh5_ctx->offset_dims[0] += 1; + + /* + * Bump the dump buffer pointer to the next time integration. + */ + bufptr += p_fbh5_ctx->tint_size; + + /* + * END write-loop for the current dump. + */ + } + + /* + * Bump counters. + */ + p_fbh5_ctx->byte_count += bufsize; + + /* + * Bye-bye. + */ +} + From 2f3c5d446b4aa69133fb7b06d8da9ec00458552b Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Mon, 15 Nov 2021 15:21:13 -0600 Subject: [PATCH 05/39] Added the fbh5*.o, HDF5 include, and HDF5 lib dependencies --- Makefile | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 26953a0..0856a6d 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,14 @@ INCDIR = $(PREFIX)/include LIBDIR = $(PREFIX)/lib DATADIR = $(PREFIX)/share +# Begin HDF5 definitions +INCDIR_H5= /usr/include/hdf5/serial/ +LIBDIR_h5= /usr/lib/x86_64-linux-gnu/hdf5/serial/ +LIBHDF5= :libhdf5.so +LIBHDF5_HL= :libhdf5_hl.so +LINKH5:= -L$(LIBDIR) -l $(LIBHDF5) -l $(LIBHDF5_HL) +# End HDF5 definitions + CUDA_DIR ?= $(CUDA_ROOT) CUDA_PATH ?= $(CUDA_DIR) @@ -13,7 +21,7 @@ CXX ?= g++ HOST_COMPILER ?= $(CXX) NVCC := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER) -CFLAGS = -ggdb -fPIC -I$(CUDA_PATH)/include +CFLAGS = -ggdb -fPIC -I$(CUDA_PATH)/include -I$(INCDIR_H5) ifdef DEBUG_CALLBACKS CFLAGS += -DDEBUG_CALLBACKS=$(DEBUG_CALLBACKS) endif @@ -66,14 +74,21 @@ rawspec_socket.o: rawspec_socket.h rawspec.h \ rawspectest.o: rawspec.h rawspec_rawutils.o: rawspec_rawutils.h hget.h +# Begin fbh5 objects +fbh5_open.o: fbh5_defs.h rawspec_callback.h rawspec_fbutils.h +fbh5_close.o: fbh5_defs.h rawspec_callback.h rawspec_fbutils.h +fbh5_write.o: fbh5_defs.h rawspec_callback.h rawspec_fbutils.h +fbh5_util.o: fbh5_defs.h rawspec_callback.h rawspec_fbutils.h +# End fbh5 objects + %.o: %.cu $(VERBOSE) $(NVCC) $(NVCC_FLAGS) -dc $(GENCODE_FLAGS) -o $@ -c $< -librawspec.so: rawspec_gpu.o rawspec_fbutils.o rawspec_rawutils.o - $(VERBOSE) $(NVCC) -shared $(NVCC_FLAGS) $(GENCODE_FLAGS) -o $@ $^ $(CUDA_STATIC_LIBS) +librawspec.so: rawspec_gpu.o rawspec_fbutils.o rawspec_rawutils.o fbh5_open.o fbh5_close.o fbh5_write.o fbh5_util.o + $(VERBOSE) $(NVCC) -shared $(NVCC_FLAGS) $(GENCODE_FLAGS) -o $@ $^ $(CUDA_STATIC_LIBS) $(LINKH5) rawspec: librawspec.so -rawspec: rawspec.o rawspec_file.o rawspec_socket.o +rawspec: rawspec.o rawspec_file.o rawspec_socket.o $(VERBOSE) $(NVCC) $(NVCC_FLAGS) $(GENCODE_FLAGS) -o $@ $^ -L. -lrawspec rawspectest: librawspec.so From acf23ede13d7ed39b6959b76eb67e5d2ee1bc3eb Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Tue, 16 Nov 2021 09:48:18 -0600 Subject: [PATCH 06/39] Integration of FBH5 --- fbh5_close.c | 13 +++-- fbh5_open.c | 34 ++++++++---- fbh5_util.c | 71 ------------------------ rawspec.c | 133 +++++++++++++++++++++++++++++++++------------ rawspec_callback.h | 2 + rawspec_file.c | 42 +++++++++----- rawspec_file.h | 5 +- 7 files changed, 158 insertions(+), 142 deletions(-) diff --git a/fbh5_close.c b/fbh5_close.c index 70091e8..a546601 100644 --- a/fbh5_close.c +++ b/fbh5_close.c @@ -49,15 +49,18 @@ void fbh5_close(fbh5_context_t * p_fbh5_ctx, int debug_callback) { status = H5Fclose(p_fbh5_ctx->file_id); if(status != 0) fbh5_oops(__FILE__, __LINE__, "fbh5_close H5Fclose FAILED\n"); + p_fbh5_ctx->active = 0; /* * Closing statistics. */ - printf("fbh5_close: Context closed.\n"); - printf("fbh5_close: %ld dumps processed.\n", p_fbh5_ctx->dump_count); - printf("fbh5_close: %lld time integrations processed.\n", p_fbh5_ctx->offset_dims[0]); - MiBstore = (double) sz_store / MILLION; - printf("fbh5_close: Compressed %.2f MiB --> %.2f MiB\n", MiBlogical, MiBstore); + if(debug_callback) { + printf("fbh5_close: Context closed.\n"); + printf("fbh5_close: %ld dumps processed.\n", p_fbh5_ctx->dump_count); + printf("fbh5_close: %lld time integrations processed.\n", p_fbh5_ctx->offset_dims[0]); + MiBstore = (double) sz_store / MILLION; + printf("fbh5_close: Compressed %.2f MiB --> %.2f MiB\n", MiBlogical, MiBstore); + } /* * Bye-bye. diff --git a/fbh5_open.c b/fbh5_open.c index 9a2783f..01b5ea2 100644 --- a/fbh5_open.c +++ b/fbh5_open.c @@ -26,11 +26,13 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p int deflate_level = 6; // 0=no deflation, 9=highest, 6=recommended H5get_libversion(&hdf5_majnum, &hdf5_minnum, &hdf5_relnum); - printf("fbh5_open: FBH5 path: %s\n", output_path); - printf("fbh5_open: HDF5 library version: %d.%d.%d\n", hdf5_majnum, hdf5_minnum, hdf5_relnum); - printf("fbh5_open: Creating dataspace dimensions using nifs=%d and nchans=%d\n", - p_fb_hdr->nifs, - p_fb_hdr->nchans); + if(debug_callback) { + printf("fbh5_open: FBH5 path: %s\n", output_path); + printf("fbh5_open: HDF5 library version: %d.%d.%d\n", hdf5_majnum, hdf5_minnum, hdf5_relnum); + printf("fbh5_open: Creating dataspace dimensions using nifs=%d and nchans=%d\n", + p_fb_hdr->nifs, + p_fb_hdr->nchans); + } /* * Validate fb_hdr: nifs, nbits, nfpc, nchans. @@ -61,9 +63,11 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p p_fbh5_ctx->offset_dims[0] = 0; p_fbh5_ctx->offset_dims[1] = 0; p_fbh5_ctx->offset_dims[2] = 0; - printf("fbh5_open: Data element byte size = %d.\n", p_fbh5_ctx->elem_size); - printf("fbh5_open: Time integration byte size = %ld.\n", p_fbh5_ctx->tint_size); - + if(debug_callback) { + printf("fbh5_open: Data element byte size = %d.\n", p_fbh5_ctx->elem_size); + printf("fbh5_open: Time integration byte size = %ld.\n", p_fbh5_ctx->tint_size); + } + /* * Open HDF5 file. Overwrite it if preexisting. */ @@ -120,10 +124,12 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p status = H5Pset_shuffle(dcpl); if(status < 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_shuffle FAILED"); - printf("fbh5_open: Using shuffle deflation.\n"); + if(debug_callback) + printf("fbh5_open: Using shuffle deflation.\n"); } else - printf("fbh5_open: Using gzip deflation.\n"); + if(debug_callback) + printf("fbh5_open: Using gzip deflation.\n"); status = H5Pset_deflate(dcpl, deflate_level); if(status < 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_deflate FAILED"); @@ -138,7 +144,8 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p status = H5Pset_chunk(dcpl, NDIMS, cdims); if(status != 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_chunk FAILED"); - printf("fbh5_open: Chunking (%lld, %lld, %lld) configured\n", cdims[0], cdims[1], cdims[2]); + if(debug_callback) + printf("fbh5_open: Chunking (%lld, %lld, %lld) configured\n", cdims[0], cdims[1], cdims[2]); /* * Define datatype for the data in the file. @@ -184,10 +191,13 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p fbh5_write_metadata(p_fbh5_ctx->dataset_id, // Dataset handle p_fb_hdr, // Metadata (SIGPROC header) debug_callback); // Tracing flag - printf("fbh5_open: Dataset metadata stored; done.\n"); + if(debug_callback) + printf("fbh5_open: Dataset metadata stored; done.\n"); /* * Bye-bye. */ + p_fbh5_ctx->active = 1; + } diff --git a/fbh5_util.c b/fbh5_util.c index 7ac16fe..821d17d 100644 --- a/fbh5_util.c +++ b/fbh5_util.c @@ -134,74 +134,3 @@ void fbh5_set_ds_label(fbh5_context_t * p_fbh5_ctx, char * label, int dims_index } } -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * hexdump.c * - * --------- * - * Cheap-o-cigar dump anything-to-hex-and-string on stdout * - * * - * Usage: * - * hexdump(label, addr, len, per_line); * - * * - * where: * - * label: label of thing being dumped. * - * thingaddr: address to start dumping from. * - * thingsz: number of bytes to dump. * - * per_line: number of bytes on each output line. * - * * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -void hexdump(const char *label, const void *thingaddr, const int thingsz, int per_line) { - // Silently ignore silly per-line values. - if(per_line < 4 || per_line > 64) per_line = 16; - - int ii; // index into buffer, byte for byte - unsigned char buffer[per_line+1]; - const unsigned char * pc = (const unsigned char *)thingaddr; - - // Output description if given. - if(label != NULL) - printf("hexdump %s (%d bytes):\n", label, thingsz); - else - printf("hexdump (no label) %d bytes\n", thingsz); - - // Length checks. - if(thingsz < 1) { - fprintf(stderr, "\nhexdump: thingsz(%d) < 1; label=%s\n", thingsz, label); - exit(86); - } - - // Process every byte in the data. - for (ii = 0; ii < thingsz; ii++) { - // Multiple of per_line means new or first line (with line offset). - if((ii % per_line) == 0) { - // Only print previous-line ASCII buffer for lines beyond first. - if(ii != 0) printf(" %s\n", buffer); - - // Output the offset of current line. - printf(" %04x ", ii); - } - - // Now the hex code for the specific character. - printf(" %02x", pc[ii]); - - // And buffer a printable ASCII character for later. - if((pc[ii] < 0x20) || (pc[ii] > 0x7e)) // isprint() may be better. - buffer[ii % per_line] = '.'; - else - buffer[ii % per_line] = pc[ii]; - - // Add a C string terminator. - buffer[(ii % per_line) + 1] = '\0'; - } - - // Pad out last line if not exactly per_line characters. - while ((ii % per_line) != 0) { - printf(" "); - ii++; - } - - // Dump the final ASCII buffer. - printf(" %s\n", buffer); -} - - diff --git a/rawspec.c b/rawspec.c index 09d04ae..cea429d 100644 --- a/rawspec.c +++ b/rawspec.c @@ -60,6 +60,7 @@ ssize_t read_fully(int fd, void * buf, size_t bytes_to_read) } static struct option long_opts[] = { + {"fbh5", 0, NULL, '5'}, {"ant", 1, NULL, 'a'}, {"batch", 0, NULL, 'b'}, {"dest", 1, NULL, 'd'}, @@ -90,6 +91,7 @@ void usage(const char *argv0) { "Usage: %s [options] STEM [...]\n" "\n" "Options:\n" + " -5, --fbh5 Format output Filterbank files as FBH5 (.h5) instead of SIGPROC(.fil)\n" " -a, --ant=ANT The 0-indexed antenna to exclusively process [-1]\n" " -b, --batch=BC Batch process BC coarse-channels at a time (1: auto, <1: disabled) [0]\n" " -d, --dest=DEST Destination directory or host:port\n" @@ -192,6 +194,9 @@ char tmp[16]; int input_conjugated = -1; int only_output_ics = 0; + // FBH5 fields + int flag_fbh5_output = 0; + // For net data rate rate calculations double rate = 6.0; double sum_inv_na; @@ -212,6 +217,10 @@ char tmp[16]; return 0; break; + case '5': // FBH5 output format requested + flag_fbh5_output = 1; + break; + case 'a': // Antenna selection to process ant = strtol(optarg, NULL, 0); break; @@ -438,8 +447,15 @@ char tmp[16]; cb_data[i].Nant = 1; // Init callback file descriptors to sentinal values - cb_data[i].fd = malloc(sizeof(int)*1); - cb_data[i].fd[0] = -1; + if(flag_fbh5_output) { + cb_data[i].flag_fbh5_output = 1; + cb_data[i].fbh5_ctx_ant = malloc(sizeof(fbh5_context_t)); + cb_data[i].fbh5_ctx_ant.active = 0; + } else { + cb_data[i].flag_fbh5_output = 0; + cb_data[i].fd = malloc(sizeof(int)); + cb_data[i].fd[0] = -1; + } } // Set output mode specific callback function @@ -574,20 +590,42 @@ char tmp[16]; // close previous for(i=0; iflag_fbh5_output) { + fbh5_open(&(cb_data->fbh5_ctx), &(cb_data->fb_hdr), fname, cb_data->debug_callback); + fd = 0; } else { - posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC, 0664); + if(fd == -1) { + perror(fname); + } else { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + } } return fd; } -int open_output_file_per_antenna_and_write_header(callback_data_t *cb_data, const char * dest, const char *stem, int output_idx) +// Open an output file used for non-antenna output or for a single antenna. +int open_output_file_per_antenna_and_write_header(callback_data_t * cb_data, const char * dest, const char * stem, int output_idx) { char ant_stem[PATH_MAX+1]; - if(cb_data->per_ant_out){ - cb_data->fb_hdr.nchans /= cb_data->Nant; - } for(int i = 0; i < (cb_data->per_ant_out ? cb_data->Nant : 1); i++){ if(cb_data->per_ant_out){ snprintf(ant_stem, PATH_MAX, "%s-ant%03d", stem, i); @@ -52,7 +56,7 @@ int open_output_file_per_antenna_and_write_header(callback_data_t *cb_data, cons snprintf(ant_stem, PATH_MAX, "%s", stem); } - cb_data->fd[i] = open_output_file(dest, ant_stem, output_idx); + cb_data->fd[i] = open_output_file(cb_data, dest, ant_stem, output_idx); if(cb_data->fd[i] == -1) { // If we can't open this output file, we probably won't be able to // open any more output files, so print message and bail out. @@ -61,10 +65,8 @@ int open_output_file_per_antenna_and_write_header(callback_data_t *cb_data, cons } // Write filterbank header to output file - fb_fd_write_header(cb_data->fd[i], &cb_data->fb_hdr); - } - if(cb_data->per_ant_out){ - cb_data->fb_hdr.nchans *= cb_data->Nant; + if(! cb_data->flag_fbh5_output) + fb_fd_write_header(cb_data->fd[i], &cb_data->fb_hdr); } return 0; } @@ -86,7 +88,17 @@ void * dump_file_thread_func(void *arg) // Assume that the following file-descriptors aren't valid break; } - write(cb_data->fd[i], cb_data->h_pwrbuf + i * ant_stride + j * pol_stride + k * spectra_stride, ant_stride * sizeof(float)); + if(cb_data->flag_fbh5_output) { + fbh5_write(&(cb_data->fbh5_ctx), + &(cb_data->fb_hdr), + cb_data->h_pwrbuf + i * ant_stride + j * pol_stride + k * spectra_stride, + ant_stride * sizeof(float), + cb_data->debug_callback); + } else { + write(cb_data->fd[i], + cb_data->h_pwrbuf + i * ant_stride + j * pol_stride + k * spectra_stride, + ant_stride * sizeof(float)); + } } } } diff --git a/rawspec_file.h b/rawspec_file.h index b34714f..2dff356 100644 --- a/rawspec_file.h +++ b/rawspec_file.h @@ -8,12 +8,11 @@ extern "C" { #endif -int open_output_file(const char * dest, const char *stem, int output_idx); +int open_ics_output_file(callback_data_t *cb_data, const char * dest, const char *stem, int output_idx); int open_output_file_per_antenna_and_write_header(callback_data_t *cb_data, const char * dest, const char *stem, int output_idx); -void dump_file_callback( - rawspec_context * ctx, int output_product, int callback_type); +void dump_file_callback(rawspec_context * ctx, int output_product, int callback_type); #ifdef __cplusplus } From ef1e7dc6b89a55acf2311befdf0fe98c570bb06e Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Tue, 16 Nov 2021 12:57:55 -0600 Subject: [PATCH 07/39] Compatibility for fd and fbh5 is not easy! --- fbh5_defs.h | 2 + rawspec.c | 38 +++++++++++------- rawspec_file.c | 106 +++++++++++++++++++++++++++++++++---------------- rawspec_file.h | 3 +- 4 files changed, 98 insertions(+), 51 deletions(-) diff --git a/fbh5_defs.h b/fbh5_defs.h index e0a068e..efb28e7 100644 --- a/fbh5_defs.h +++ b/fbh5_defs.h @@ -36,6 +36,7 @@ #define NDIMS 3 // # of data matrix dimensions (rank) #define FILTERBANK_CLASS "FILTERBANK" // File-level attribute "CLASS" #define FILTERBANK_VERSION "2.0" // File-level attribute "VERSION" +#define ENABLER_FD_FOR_FBH5 42 // Fake fd value to enable dump_file_thread_func() /* * fbh5 API functions @@ -60,3 +61,4 @@ void fbh5_set_ds_label(fbh5_context_t * p_fbh5_ctx, char * label, int dims_in void hexdump(const char *label, const void *thingaddr, const int thingsz, int per_line); #endif + diff --git a/rawspec.c b/rawspec.c index cea429d..077f820 100644 --- a/rawspec.c +++ b/rawspec.c @@ -18,10 +18,10 @@ #include "rawspec.h" #include "rawspec_file.h" #include "rawspec_socket.h" -#include "rawspec_callback.h" #include "rawspec_version.h" #include "rawspec_rawutils.h" #include "rawspec_fbutils.h" +#include "fbh5_defs.h" #define ELAPSED_NS(start,stop) \ (((int64_t)stop.tv_sec-start.tv_sec)*1000*1000*1000+(stop.tv_nsec-start.tv_nsec)) @@ -450,7 +450,7 @@ char tmp[16]; if(flag_fbh5_output) { cb_data[i].flag_fbh5_output = 1; cb_data[i].fbh5_ctx_ant = malloc(sizeof(fbh5_context_t)); - cb_data[i].fbh5_ctx_ant.active = 0; + cb_data[i].fbh5_ctx_ant[0].active = 0; } else { cb_data[i].flag_fbh5_output = 0; cb_data[i].fd = malloc(sizeof(int)); @@ -790,19 +790,26 @@ char tmp[16]; // Open one or more output files. // Handle both per-antenna output and single file output. if(!only_output_ics) { - if(open_output_file_per_antenna_and_write_header(&cb_data[i], dest, argv[si], outidx + i) != 0)) + // Open nants=0 case or open all of the antennas. + int retcode = open_output_file_per_antenna_and_write_header(&cb_data[i], + dest, + argv[si], + outidx + i); + if(retcode != 0) return 1; // give up } // Handle ICS. - if(ctx.incoherently_sum){ - cb_data[i].fd_ics = open_ics_output_file(&cb_data[i], dest, ics_output_stem, outidx + i)); - if(! flag_fbh5_output) { - if(cb_data[i].fd_ics == -1) { - // If we can't open this output file, we probably won't be able to - // open any more output files, so print message and bail out. - fprintf(stderr, "cannot open output file, giving up\n"); - return 1; // Give up - } + if(ctx.incoherently_sum) { + cb_data[i].fd_ics = open_output_file(&cb_data[i], + dest, + ics_output_stem, + outidx + i, + /* ICS */ -1); + if(cb_data[i].fd_ics == -1) { + // If we can't open this output file, we probably won't be able to + // open any more output files, so print message and bail out. + fprintf(stderr, "cannot open output file, giving up\n"); + return 1; // Give up } // Write filterbank header to SIGPROC output ICS file. @@ -810,9 +817,9 @@ char tmp[16]; if(! flag_fbh5_output) { fb_fd_write_header(cb_data[i].fd_ics, &cb_data[i].fb_hdr); } - } - } - } + } // if(ctx.incoherently_sum) + } // if(output_mode == RAWSPEC_FILE) + } // for(i=0; i #include "rawspec_file.h" +#include "fbh5_defs.h" -// Open an output file used for ICS only. -int open_ics_output_file(callback_data_t * cb_data, const char * dest, const char * stem, int output_idx) +// Open a single Filterbank file for one of the following: +// * nants = 0 +// * a single antenna of a set +// * ICS +int open_output_file(callback_data_t *cb_data, const char * dest, const char *stem, int output_idx, int antenna_index) { int fd; const char * basename; @@ -31,32 +35,42 @@ int open_ics_output_file(callback_data_t * cb_data, const char * dest, const cha } fname[PATH_MAX] = '\0'; if(cb_data->flag_fbh5_output) { - fbh5_open(&(cb_data->fbh5_ctx), &(cb_data->fb_hdr), fname, cb_data->debug_callback); - fd = 0; + // Open an FBH5 output file. + // If antenna_index < 0, then use the ICS context; + // Else, use the indicated antenna context. + if(antenna_index < 0) + fbh5_open(&(cb_data->fbh5_ctx_ics), &(cb_data->fb_hdr), fname, cb_data->debug_callback); + else + fbh5_open(&(cb_data->fbh5_ctx_ant[antenna_index]), &(cb_data->fb_hdr), fname, cb_data->debug_callback); + return ENABLER_FD_FOR_FBH5; + } + + // Open a SIGPROC Filterbank output file. + fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC, 0664); + if(fd == -1) { + perror(fname); } else { - fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC, 0664); - if(fd == -1) { - perror(fname); - } else { - posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); - } + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); } return fd; } -// Open an output file used for non-antenna output or for a single antenna. -int open_output_file_per_antenna_and_write_header(callback_data_t * cb_data, const char * dest, const char * stem, int output_idx) +// Open one or more output Filterbank files for the following cases: +// * nants = 0 +// * the set of antennas when nants > 0 +int open_output_file_per_antenna_and_write_header(callback_data_t *cb_data, const char * dest, const char * stem, int output_idx) { char ant_stem[PATH_MAX+1]; - for(int i = 0; i < (cb_data->per_ant_out ? cb_data->Nant : 1); i++){ - if(cb_data->per_ant_out){ + + for(int i = 0; i < (cb_data->per_ant_out ? cb_data->Nant : 1); i++) { + if(cb_data->per_ant_out) { snprintf(ant_stem, PATH_MAX, "%s-ant%03d", stem, i); } - else{ + else { snprintf(ant_stem, PATH_MAX, "%s", stem); } - cb_data->fd[i] = open_output_file(cb_data, dest, ant_stem, output_idx); + cb_data->fd[i] = open_output_file(cb_data, dest, ant_stem, output_idx, i); if(cb_data->fd[i] == -1) { // If we can't open this output file, we probably won't be able to // open any more output files, so print message and bail out. @@ -64,7 +78,7 @@ int open_output_file_per_antenna_and_write_header(callback_data_t * cb_data, con return 1; // Give up } - // Write filterbank header to output file + // Write filterbank header to output file if SIGPROC. if(! cb_data->flag_fbh5_output) fb_fd_write_header(cb_data->fd[i], &cb_data->fb_hdr); } @@ -75,8 +89,8 @@ void * dump_file_thread_func(void *arg) { callback_data_t * cb_data = (callback_data_t *)arg; - if(cb_data->fd && cb_data->h_pwrbuf){ - if(cb_data->per_ant_out){ + if(cb_data->fd && cb_data->h_pwrbuf) { + if(cb_data->per_ant_out) { size_t spectra_stride = cb_data->h_pwrbuf_size / (cb_data->Nds * sizeof(float)); size_t pol_stride = spectra_stride / cb_data->fb_hdr.nifs; size_t ant_stride = pol_stride / cb_data->Nant; @@ -89,27 +103,47 @@ void * dump_file_thread_func(void *arg) break; } if(cb_data->flag_fbh5_output) { - fbh5_write(&(cb_data->fbh5_ctx), - &(cb_data->fb_hdr), - cb_data->h_pwrbuf + i * ant_stride + j * pol_stride + k * spectra_stride, + fbh5_write(&(cb_data->fbh5_ctx_ant[i]), + &(cb_data->fb_hdr), + cb_data->h_pwrbuf + i * ant_stride + j * pol_stride + k * spectra_stride, ant_stride * sizeof(float), cb_data->debug_callback); } else { - write(cb_data->fd[i], - cb_data->h_pwrbuf + i * ant_stride + j * pol_stride + k * spectra_stride, + write(cb_data->fd[i], + cb_data->h_pwrbuf + i * ant_stride + j * pol_stride + k * spectra_stride, ant_stride * sizeof(float)); - } - } - } - } - } - else{ - write(cb_data->fd[0], cb_data->h_pwrbuf, cb_data->h_pwrbuf_size); - } - } + } // if(cb_data->flag_fbh5_output) + } // for(size_t i = 0; i < cb_data->Nant; i++) + } // for(size_t j = 0; j < cb_data->fb_hdr.nifs; j++) + } // for(size_t k = 0; k < cb_data->Nds; k++) + } // if(cb_data->per_ant_out) + else { // nants = 0; single output file + if(cb_data->flag_fbh5_output) { + fbh5_write(&(cb_data->fbh5_ctx_ant[0]), + &(cb_data->fb_hdr), + cb_data->h_pwrbuf, + cb_data->h_pwrbuf_size, + cb_data->debug_callback); + } else { + write(cb_data->fd[0], + cb_data->h_pwrbuf, + cb_data->h_pwrbuf_size); + } // if(cb_data->flag_fbh5_output) + } // if(cb_data->per_ant_out) ... else + } // if(cb_data->fd && cb_data->h_pwrbuf) - if(cb_data->fd_ics && cb_data->h_icsbuf){ - write(cb_data->fd_ics, cb_data->h_icsbuf, cb_data->h_pwrbuf_size/cb_data->Nant); + if(cb_data->fd_ics && cb_data->h_icsbuf) { + if(cb_data->flag_fbh5_output) { + fbh5_write(&(cb_data->fbh5_ctx_ics), + &(cb_data->fb_hdr), + cb_data->h_icsbuf, + cb_data->h_pwrbuf_size/cb_data->Nant, + cb_data->debug_callback); + } else { + write(cb_data->fd_ics, + cb_data->h_icsbuf, + cb_data->h_pwrbuf_size/cb_data->Nant); + } } // Increment total spectra counter for this output product @@ -138,6 +172,7 @@ void dump_file_callback( cb_data->output_thread_valid = 0; } } else if(callback_type == RAWSPEC_CALLBACK_POST_DUMP) { + #ifdef VERBOSE fprintf(stderr, "cb %d writing %lu bytes:", output_product, ctx->h_pwrbuf_size[output_product]); @@ -147,6 +182,7 @@ void dump_file_callback( } fprintf(stderr, "\n"); #endif // VERBOSE + if((rc=pthread_create(&cb_data->output_thread, NULL, dump_file_thread_func, cb_data))) { fprintf(stderr, "pthread_create: %s\n", strerror(rc)); diff --git a/rawspec_file.h b/rawspec_file.h index 2dff356..118c146 100644 --- a/rawspec_file.h +++ b/rawspec_file.h @@ -8,7 +8,7 @@ extern "C" { #endif -int open_ics_output_file(callback_data_t *cb_data, const char * dest, const char *stem, int output_idx); +int open_output_file(callback_data_t *cb_data, const char * dest, const char *stem, int output_idx, int ics_flag); int open_output_file_per_antenna_and_write_header(callback_data_t *cb_data, const char * dest, const char *stem, int output_idx); @@ -19,3 +19,4 @@ void dump_file_callback(rawspec_context * ctx, int output_product, int callback_ #endif #endif // _RAWSPEC_FILE_H_ + From a1853ceb524fe4ce8242991ef9f3fff3f3359c9d Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Tue, 16 Nov 2021 13:02:21 -0600 Subject: [PATCH 08/39] int active; <---- semicolon. This isn't Python, Richard! --- rawspec_callback.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rawspec_callback.h b/rawspec_callback.h index 3982171..5dc8cdd 100644 --- a/rawspec_callback.h +++ b/rawspec_callback.h @@ -6,7 +6,7 @@ #include "rawspec_fbutils.h" typedef struct { - int active // Still active? 1=yes, 0=no + int active; // Still active? 1=yes, 0=no hid_t file_id; // File-level handle (similar to an fd) hid_t dataset_id; // Dataset "data" handle hid_t dataspace_id; // Dataspace handle for dataset "data" From bc554184868752f33626e5272f20a4bfc0c6f364 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Tue, 16 Nov 2021 14:20:16 -0600 Subject: [PATCH 09/39] Successfully wrote its first fbh5 file --- rawspectest.c | 1153 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 1032 insertions(+), 121 deletions(-) diff --git a/rawspectest.c b/rawspectest.c index b12e46d..51d2bbc 100644 --- a/rawspectest.c +++ b/rawspectest.c @@ -1,172 +1,1083 @@ +#define _GNU_SOURCE 1 + #include #include #include #include +#include #include +#include #include - -#include -#include +#include +#include +#include +#include +#include +#include #include "rawspec.h" +#include "rawspec_file.h" +#include "rawspec_socket.h" +#include "rawspec_version.h" +#include "rawspec_rawutils.h" +#include "rawspec_fbutils.h" +#include "fbh5_defs.h" #define ELAPSED_NS(start,stop) \ (((int64_t)stop.tv_sec-start.tv_sec)*1000*1000*1000+(stop.tv_nsec-start.tv_nsec)) -void -dump_callback(rawspec_context * ctx, int output_product, int callback_type) +// This stringification trick is from "info cpp" +#define STRINGIFY1(s) #s +#define STRINGIFY(s) STRINGIFY1(s) + +#ifndef DEBUG_CALLBACKS +#define DEBUG_CALLBACKS (0) +#endif + +// Reads `bytes_to_read` bytes from `fd` into the buffer pointed to by `buf`. +// Returns the total bytes read or -1 on error. A non-negative return value +// will be less than `bytes_to_read` only of EOF is reached. +ssize_t read_fully(int fd, void * buf, size_t bytes_to_read) { - printf("cb %d\n", output_product); + ssize_t bytes_read; + ssize_t total_bytes_read = 0; + + while(bytes_to_read > 0) { + bytes_read = read(fd, buf, bytes_to_read); + if(bytes_read <= 0) { + if(bytes_read == 0) { + break; + } else { + return -1; + } + } + buf += bytes_read; + bytes_to_read -= bytes_read; + total_bytes_read += bytes_read; + } + + return total_bytes_read; } -void show_gpu_memory_usage(int gpu_id) { - size_t in_use, available, total; - int gpu_id_selected; - cudaSetDevice(gpu_id); - cudaGetDevice(&gpu_id_selected); - cudaMemGetInfo(&available, &total); - in_use = total - available; - printf("GPU %d memory, in use: %ld MiB, free: %ld MiB, total: %ld MiB\n", - gpu_id_selected, - (long) ((float) in_use / 1.0e6), - (long) ((float) available / 1.0e6), - (long) ((float) total / 1.0e6)); +static struct option long_opts[] = { + {"ant", 1, NULL, 'a'}, + {"batch", 0, NULL, 'b'}, + {"dest", 1, NULL, 'd'}, + {"ffts", 1, NULL, 'f'}, + {"gpu", 1, NULL, 'g'}, + {"help", 0, NULL, 'h'}, + {"hdrs", 0, NULL, 'H'}, + {"ICS", 1, NULL, 'I'}, + {"fbh5", 0, NULL, 'j'}, + {"nchan", 1, NULL, 'n'}, + {"outidx", 1, NULL, 'o'}, + {"pols", 1, NULL, 'p'}, + {"rate", 1, NULL, 'r'}, + {"schan", 1, NULL, 's'}, + {"splitant",0, NULL, 'S'}, + {"ints", 1, NULL, 't'}, + {"version", 0, NULL, 'v'}, + {"debug", 0, NULL, 'z'}, + {0,0,0,0} +}; + +void usage(const char *argv0) { + const char * bname = basename(argv0); + // Should "never" happen + if(!bname) { + bname = argv0; + } + + fprintf(stderr, + "Usage: %s [options] STEM [...]\n" + "\n" + "Options:\n" + " -a, --ant=ANT The 0-indexed antenna to exclusively process [-1]\n" + " -b, --batch=BC Batch process BC coarse-channels at a time (1: auto, <1: disabled) [0]\n" + " -d, --dest=DEST Destination directory or host:port\n" + " -f, --ffts=N1[,N2...] FFT lengths [1048576, 8, 1024]\n" + " -g, --GPU=IDX Select GPU device to use [0]\n" + " -H, --hdrs Save headers to separate file\n" + " -i, --ics=W1[,W2...] Output incoherent-sum (exclusively, unless with -S)\n" + " specifying per antenna-weights or a singular, uniform weight\n" + " -j, --fbh5 Format output Filterbank files as FBH5 (.h5) instead of SIGPROC(.fil)\n" + " -n, --nchan=N Number of coarse channels to process [all]\n" + " -o, --outidx=N First index number for output files [0]\n" + " -p --pols={1|4}[,...] Number of output polarizations [1]\n" + " 1=total power, 4=cross pols, -4=full stokes\n" + " -r, --rate=GBPS Desired net data rate in Gbps [6.0]\n" + " -s, --schan=C First coarse channel to process [0]\n" + " -S, --splitant Split output into per antenna files\n" + " -t, --ints=N1[,N2...] Spectra to integrate [51, 128, 3072]\n" + " -z, --debug Turn on selected debug output\n" + "\n" + " -h, --help Show this message\n" + " -v, --version Show version and exit\n" + , bname + ); } -int main(int argc, char * argv[]) +int open_headers_file(const char * dest, const char *stem) { + int fd; + const char * basename; + char fname[PATH_MAX+1]; + + // If dest is given and it's not empty + if(dest && dest[0]) { + // Look for last '/' in stem + basename = strrchr(stem, '/'); + if(basename) { + // If found, advance beyond it to first char of basename + basename++; + } else { + // If not found, use stem as basename + basename = stem; + } + snprintf(fname, PATH_MAX, "%s/%s.rawspec.headers", dest, basename); + } else { + snprintf(fname, PATH_MAX, "%s.rawspec.headers", stem); + } + fname[PATH_MAX] = '\0'; + fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC, 0664); + if(fd == -1) { + perror(fname); + } else { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + } + return fd; +} + +int main(int argc, char *argv[]) +{ + int si; // Indexes the stems + int fi; // Indexes the files for a given stem + int bi; // Counts the blocks processed for a given file int i; - int j; - rawspec_context ctx = {0}; +int j, k; +char tmp[16]; + void * pv; + int fdin; + int fdhdrs = -1; + int next_stem = 0; + int save_headers = 0; + int per_ant_out = 0; + unsigned int Nc; // Number of coarse channels across the observation (possibly multi-antenna) + unsigned int Ncpa;// Number of coarse channels per antenna + unsigned int Np; // Number of polarizations + unsigned int Ntpb; // Number of time samples per block + unsigned int Nbps; // Number of bits per sample + uint64_t block_byte_length; // Compute the length once + char expand4bps_to8bps; // Expansion flag + int64_t pktidx0; + int64_t pktidx; + int64_t dpktidx; + char fname[PATH_MAX+1]; + int opt; + char * argv0; + char * pchar; + char * bfname; + char * dest = NULL; // default output dest is same place as input stem + char * ics_output_stem = NULL; + rawspec_output_mode_t output_mode = RAWSPEC_FILE; + char * dest_port = NULL; // dest port for network output + int fdout; + int open_flags; + size_t bytes_read; + size_t total_bytes_read; + off_t pos; + rawspec_raw_hdr_t raw_hdr; + callback_data_t cb_data[MAX_OUTPUTS]; + rawspec_context ctx; + int ant = -1; + unsigned int schan = 0; + unsigned int nchan = 0; + unsigned int outidx = 0; + int input_conjugated = -1; + int only_output_ics = 0; - // Timing variables - struct timespec ts_start, ts_stop; - uint64_t elapsed_ns=0; + // Selected dynamic debugging + int flag_debugging = 0; - int blocsize = 92274688; - size_t nfine; + // FBH5 fields + int flag_fbh5_output = 0; - // Show librawspec version on startup - printf("using librawspec %s\n", rawspec_version_string()); - - ctx.No = 4; - ctx.Np = 2; - ctx.Nc = 88; - ctx.Nbc = 1; - ctx.Nbps = 8; - if(argc > 1) { - ctx.Nbps = strtoul(argv[1], NULL, 0); - if(ctx.Nbps == 0) { - ctx.Nbps = 8; + // For net data rate rate calculations + double rate = 6.0; + double sum_inv_na; + uint64_t total_packets = 0; + uint64_t total_bytes = 0; + uint64_t total_ns = 0; + + // Init rawspec context + memset(&ctx, 0, sizeof(ctx)); + ctx.Npolout[0] = 1; // others will be set later + + // Parse command line. + argv0 = argv[0]; + while((opt=getopt_long(argc,argv,"a:b:d:f:g:HI:i:n:o:p:r:Ss:t:hv",long_opts,NULL))!=-1) { + switch (opt) { + case 'h': // Help + usage(argv0); + return 0; + break; + + case 'j': // FBH5 output format requested + flag_fbh5_output = 1; + break; + + case 'z': // Selected dynamic debugging + flag_debugging = 1; + break; + + case 'a': // Antenna selection to process + ant = strtol(optarg, NULL, 0); + break; + + case 'b': // Batch-channels + ctx.Nbc = strtol(optarg, NULL, 0); + break; + + case 'd': // Output destination + dest = optarg; + // If dest contains at least one ':', it's HOST:PORT and we're + // outputting over the network. + pchar = strrchr(dest, ':'); + if(pchar) { + // NUL terminate hostname, advance to port + *pchar++ = '\0'; + dest_port = pchar; + output_mode = RAWSPEC_NET; + } + break; + + case 'f': // Fine channel(s) per coarse channel + for(i=0, pchar = strtok(optarg,","); + pchar != NULL; i++, pchar = strtok(NULL, ",")) { + if(i>=MAX_OUTPUTS){ + fprintf(stderr, + "error: up to %d fine channel counts supported.\n", MAX_OUTPUTS); + return 1; + } + ctx.Nts[i] = strtoul(pchar, NULL, 0); + } + // If no comma (i.e. single value) + if(i==0) { + ctx.Nts[0] = strtoul(optarg, NULL, 0); + } + break; + + case 'g': // GPU device to use + ctx.gpu_index = strtol(optarg, NULL, 0); + printf("using requested GPU: %d\n", ctx.gpu_index); + break; + + case 'H': // Save headers + save_headers = 1; + break; + + case 'i': // Incoherently sum exclusively + only_output_ics = 1; + ctx.incoherently_sum = 1; + ctx.Naws = 1; + // Count number of + for(i=0; i < strlen(optarg); i++) + ctx.Naws += optarg[i]==','; + + char *weight_end; + ctx.Aws = malloc(ctx.Naws*sizeof(float)); + + for(i=0; i < ctx.Naws; i++){ + ctx.Aws[i] = strtof(optarg, &weight_end); + optarg = weight_end; + } + + break; + + case 'n': // Number of coarse channels to process + nchan = strtoul(optarg, NULL, 0); + break; + + case 'o': // Index number for first output product file name + outidx = strtoul(optarg, NULL, 0); + break; + + case 'p': // Number of pol products to output + for(i=0, pchar = strtok(optarg,","); + pchar != NULL; i++, pchar = strtok(NULL, ",")) { + if(i>=MAX_OUTPUTS){ + fprintf(stderr, + "error: up to %d pol modes supported.\n", MAX_OUTPUTS); + return 1; + } + ctx.Npolout[i] = strtoul(pchar, NULL, 0); + } + // If no comma (i.e. single value) + if(i==0) { + ctx.Npolout[0] = strtoul(optarg, NULL, 0); + } + break; + + case 'r': // Relative rate to send packets + rate = strtod(optarg, NULL); + break; + + case 's': // First coarse channel to process + schan = strtoul(optarg, NULL, 0); + break; + + case 'S': // Split output per antenna + per_ant_out = 1; + break; + + case 't': // Number of spectra to accumumate + for(i=0, pchar = strtok(optarg,","); + pchar != NULL; i++, pchar = strtok(NULL, ",")) { + if(i>=MAX_OUTPUTS){ + fprintf(stderr, + "error: up to %d integration counts supported.\n", MAX_OUTPUTS); + return 1; + } + ctx.Nas[i] = strtoul(pchar, NULL, 0); + } + // If no comma (i.e. single value) + if(i==0) { + ctx.Nas[0] = strtoul(optarg, NULL, 0); + } + break; + + case 'v': // Version + printf("rawspec %s\n", STRINGIFY(RAWSPEC_VERSION)); + printf("librawspec %s\n", rawspec_version_string()); + return 0; + break; + + case '?': // Command line parsing error + default: + printf("Unknown CLI option '%c'\n", opt); + usage(argv0); + return 1; + break; } } - printf("using %u bits per sample\n", ctx.Nbps); - ctx.Ntpb = blocsize / (2 * ctx.Np * ctx.Nc * ctx.Nbps/8); - ctx.Nts[0] = (1<<20); - ctx.Nts[1] = (1<<3); - ctx.Nts[2] = (1<<10); - ctx.Nts[3] = (1<<10); - // One dump per output product - ctx.Nas[0] = (1<<(20 - 20)); - ctx.Nas[1] = (1<<(20 - 3)); - ctx.Nas[2] = (1<<(20 - 10)); - ctx.Nas[3] = (1<<(20 - 10)); - // Auto-calculate Nb/Nb_host and let library manage input block buffers - ctx.Nb = 0; - ctx.Nb_host = 0; - ctx.h_blkbufs = NULL; - // Dump callback - ctx.dump_callback = dump_callback; - // Pol modes for each output product - ctx.Npolout[0] = 1; - ctx.Npolout[1] = 1; - ctx.Npolout[2] = 4; - ctx.Npolout[3] = -4; - - ctx.gpu_index = 0; - ctx.input_conjugated = 1; - - // Initialize - if(rawspec_initialize(&ctx)) { - fprintf(stderr, "initialization failed\n"); + + // Skip past option args + argc -= optind; + argv += optind; + + // If no stems given, print usage and exit + if(argc == 0) { + usage(argv0); return 1; } - printf("initialization succeeded, RAWSPEC_BLOCSIZE=%u\n", - RAWSPEC_BLOCSIZE(&ctx)); - // Setup input data - for(i=0; i 0) { + // Copy value from previous output product + ctx.Npolout[i] = ctx.Npolout[i-1]; + } else if(ctx.Npolout[i]!=1 && abs(ctx.Npolout[i])!=4) { + fprintf(stderr, + "error: number of output pols must be 1 or +/- 4\n"); + return 1; + } - printf("copied %u bytes in %.6f sec (%.3f GBps)\n", - blocsize * ctx.Nb, - elapsed_ns / 1e9, - blocsize * ctx.Nb / (double)elapsed_ns); + // Full-pol mode is not supported for network output + if(ctx.Npolout[i] != 1 && output_mode != RAWSPEC_FILE) { + fprintf(stderr, + "error: full-pol mode is not supported for network output\n"); + return 1; + } } - printf("starting processing\n"); - - for(i=0; i<4; i++) { - clock_gettime(CLOCK_MONOTONIC, &ts_start); + // Init user_data to be array of callback data structures + ctx.user_data = &cb_data; - rawspec_start_processing(&ctx, -1); - rawspec_wait_for_completion(&ctx); + // Zero-out the callback data sructures. + // Turn on dynamic debugging if requested. + for(i=0; i 1 && !(per_ant_out || ctx.incoherently_sum)){ + printf("NANTS = %d >1: Enabling --split-ant in lieu of neither --split-ant nor --ics flags.\n", raw_hdr.nants); + per_ant_out = 1; + } + + // If splitting output per antenna, re-alloc the fd array. + if(per_ant_out) { + if(output_mode == RAWSPEC_FILE){ + if(ant != -1){ + printf("Ignoring --ant %d option:\n\t", ant); + } + printf("Splitting output per %d antennas\n", + raw_hdr.nants); + // close previous + for(i=0; i raw_hdr.nants - 1 || ant < 0) { + printf("bad antenna selection: ant <> {0, nants} (%u <> {0, %d})\n", + ant, raw_hdr.nants); + close(fdin); + break; // Goto next stem + } + if(schan >= Ncpa) { + printf("bad schan specification with antenna selection: " + "schan > antnchan {obsnchan/nants} (%u > %u {%d/%d})\n", + schan, Ncpa, raw_hdr.obsnchan, raw_hdr.nants); + close(fdin); + break; // Goto next stem + } + + // Set Nc to Ncpa and skip previous antennas + printf("Selection of antenna %d equates to a starting channel of %d\n", ant, ant*Ncpa); + schan += ant * Ncpa; + Nc = Ncpa; + } + + // If processing a subset of coarse channels + if(nchan != 0) { + // Validate schan and nchan + if(ant == -1 && // no antenna selection + (schan + nchan > Nc)) { + + printf("bad channel range: schan + nchan > obsnchan (%u + %u > %d)\n", + schan, nchan, raw_hdr.obsnchan); + close(fdin); + break; // Goto next stem + } + else if(ant != -1 && // antenna selection + (schan + nchan > (ant + 1) * Ncpa)) { + printf("bad channel range: schan + nchan > antnchan {obsnchan/nants} (%u + %u > %d {%d/%d})\n", + schan - ant * Ncpa, nchan, Ncpa, raw_hdr.obsnchan, raw_hdr.nants); + close(fdin); + break; // Goto next stem + } + // Use nchan as Nc + Nc = nchan; + } + + // Determine if input is conjugated + input_conjugated = (raw_hdr.obsbw < 0) ? 1 : 0; + + // If block dimensions or input conjugation have changed + if(Nc != ctx.Nc || Np != ctx.Np || Nbps != ctx.Nbps || Ntpb != ctx.Ntpb + || input_conjugated != ctx.input_conjugated) { + // Cleanup previous block, if it has been initialized + if(ctx.Ntpb != 0) { + rawspec_cleanup(&ctx); + } + // Remember new dimensions and input conjugation + ctx.Nant = raw_hdr.nants; + ctx.Nc = Nc; + ctx.Np = Np; + ctx.Ntpb = Ntpb; + ctx.Nbps = Nbps; + ctx.input_conjugated = input_conjugated; + + // Initialize for new dimensions and/or conjugation + ctx.Nb = 0; // auto-calculate + ctx.Nb_host = 0; // auto-calculate + ctx.h_blkbufs = NULL; // auto-allocate + if(rawspec_initialize(&ctx)) { + fprintf(stderr, "rawspec initialization failed\n"); + close(fdin); + // Forget new dimensions + ctx.Nant = 0; + ctx.Nc = 0; + ctx.Np = 0; + ctx.Ntpb = 0; + ctx.Nbps = 0; + return 1; // fixes issue #23 + } else { + // printf("initialization succeeded for new block dimensions\n"); + block_byte_length = (2 * ctx.Np * ctx.Nc * ctx.Nbps)/8 * ctx.Ntpb; + + // The GPU supports only 8bit and 16bit sample bit-widths. The strategy + // for handling 4bit samples is to expand them out to 8bits, and there-onwards + // use the expanded 8bit samples. The device side rawspec_initialize actually still + // complains about the indication of the samples being 4bits. But the + // expand4bps_to8bps flag is used to call rawspec_copy_blocks_to_gpu_expanding_complex4, + // leading to the samples being expanded before any device side computation happens + // in rawspec_start_processing. The ctx.Nbps is left as 8. + if (ctx.Nbps == 8 && Nbps == 4){ + printf("CUDA memory initialised for %d bits per sample,\n\t" + "will expand header specified %d bits per sample.\n", ctx.Nbps, Nbps); + expand4bps_to8bps = 1; + } + + // Copy fields from ctx to cb_data + for(i=0; idebug_callback) + printf("rawspec-main: open_output_file_per_antenna_and_write_header - successful\n"); + } + // Handle ICS. + if(ctx.incoherently_sum) { + cb_data[i].fd_ics = open_output_file(&cb_data[i], + dest, + ics_output_stem, + outidx + i, + /* ICS */ -1); + if(cb_data[i].fd_ics == -1) { + // If we can't open this output file, we probably won't be able to + // open any more output files, so print message and bail out. + fprintf(stderr, "cannot open output file, giving up\n"); + return 1; // Give up + if(cb_data->debug_callback) + printf("rawspec-main: open_output_file - successful\n"); + } + + // Write filterbank header to SIGPROC output ICS file. + // If FBH5, the header was already written by fbh5_open(). + if(! flag_fbh5_output) { + fb_fd_write_header(cb_data[i].fd_ics, &cb_data[i].fb_hdr); + } + } // if(ctx.incoherently_sum) + } // if(output_mode == RAWSPEC_FILE) + } // for(i=0; i pktidx) { + dpktidx = raw_hdr.pktidx - pktidx; + } + + // Handle cases were the current pktidx is not the expected distance + // from the previous pktidx. + if(raw_hdr.pktidx - pktidx != dpktidx) { + // Cannot go backwards or forwards by non-multiple of dpktidx + if(raw_hdr.pktidx < pktidx) { + printf("got backwards jump in pktidx: %ld -> %ld\n", + pktidx, raw_hdr.pktidx); + // Give up on this stem and go to next stem + next_stem = 1; + break; + } else if((raw_hdr.pktidx - pktidx) % dpktidx != 0) { + printf("got misaligned jump in pktidx: (%ld - %ld) %% %ld != 0\n", + raw_hdr.pktidx, pktidx, dpktidx); + // Give up on this stem and go to next stem + next_stem = 1; + break; + } else if (raw_hdr.pktidx == pktidx ){ + printf("got null jump in pktidx: (%ld - %ld) == 0\n", + raw_hdr.pktidx, pktidx); + // just skip this block + break; + } + + // Put in filler blocks of zeros + while(raw_hdr.pktidx - pktidx != dpktidx) { + // Increment pktidx to next missing value + pktidx += dpktidx; + + // Fill block buffer with zeros + memset(ctx.h_blkbufs[bi%ctx.Nb_host], 0, raw_hdr.blocsize); + +#ifdef VERBOSE + fprintf(stderr, "%3d %016lx:", bi, pktidx); + fprintf(stderr, " -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --\n"); +#endif // VERBOSE + + // If this is the last block of an input buffer, start processing + if(bi % ctx.Nb == ctx.Nb - 1) { +#ifdef VERBOSE + fprintf(stderr, "block %3d buf 0: ", bi); + for(j=0; j<16; j++) { + fprintf(stderr, " %02x", ctx.h_blkbufs[0][j] & 0xff); + } + fprintf(stderr, "\n"); +#endif // VERBOSE + rawspec_wait_for_completion(&ctx); + rawspec_copy_blocks_to_gpu_and_start_processing(&ctx, ctx.Nb, expand4bps_to8bps, RAWSPEC_FORWARD_FFT); + } + + // Increment block counter + bi++; + } // filler zero blocks + } // irregular pktidx step + + // Seek past first schan channel + lseek(fdin, (2 * ctx.Np * schan * Nbps)/8 * ctx.Ntpb, SEEK_CUR); + + // Read ctx.Nc coarse channels from this block + bytes_read = read_fully(fdin, + ctx.h_blkbufs[bi % ctx.Nb_host], + (expand4bps_to8bps ? block_byte_length/2 : block_byte_length)); + + // Seek past channels after schan+nchan + lseek(fdin, (2 * ctx.Np * (raw_hdr.obsnchan-(schan+Nc)) * Nbps)/8 * ctx.Ntpb, SEEK_CUR); + + if(bytes_read == -1) { + perror("read"); + next_stem = 1; + break; // Goto next file + } else if(bytes_read < (expand4bps_to8bps ? block_byte_length/2 : block_byte_length)) { + fprintf(stderr, "incomplete block at EOF\n"); + next_stem = 1; + break; // Goto next file + } + total_bytes_read += bytes_read; + +#ifdef VERBOSE + fprintf(stderr, "%3d %016lx:", bi, raw_hdr.pktidx); + for(j=0; j<16; j++) { + fprintf(stderr, " %02x", ctx.h_blkbufs[bi%ctx.Nb_host][j] & 0xff); + } + fprintf(stderr, "\n"); +#endif // VERBOSE + + // If this is the last block of an input buffer, start processing + if(bi % ctx.Nb == ctx.Nb - 1) { +#ifdef VERBOSE + fprintf(stderr, "block %3d buf 0: ", bi); + for(j=0; j<16; j++) { + fprintf(stderr, " %02x", ctx.h_blkbufs[0][j] & 0xff); + } + fprintf(stderr, "\n"); +#endif // VERBOSE + rawspec_wait_for_completion(&ctx); + rawspec_copy_blocks_to_gpu_and_start_processing(&ctx, ctx.Nb, expand4bps_to8bps, RAWSPEC_FORWARD_FFT); + } + + // Remember pktidx + pktidx = raw_hdr.pktidx; + + // Increment block index to next block (which may be in the next file) + bi++; + + // Read obs params of next block + pos = rawspec_raw_read_header(fdin, &raw_hdr); + if(pos <= 0) { + if(pos == -1) { + fprintf(stderr, "error getting obs params from %s [%s]\n", + fname, strerror(errno)); + } + break; + } + } // For each block + + // Done with input file + close(fdin); + + // If skipping to next stem + if(next_stem) { + next_stem = 0; + // break out of each file loop + break; + } + } // each file for stem + + // Wait for GPU work to complete + if(ctx.Nc) { + rawspec_wait_for_completion(&ctx); + } + + // Close output files + if(output_mode == RAWSPEC_FILE) { + for(i=0; i 0) { + printf(" (%u packets, %.3f Gbps)", cb_data[i].total_packets, + 8.0 * cb_data[i].total_bytes / cb_data[i].total_ns); + + total_packets += cb_data[i].total_packets; + total_bytes += cb_data[i].total_bytes; + total_ns += cb_data[i].total_ns; + } + printf("\n"); + } + + if(total_ns > 0) { + printf("combined total : %lu packets, %.3f Gbps\n", + total_packets, 8.0 * total_bytes / total_ns); + } return 0; } + + From 093b2deadaf126e735cbcefd9278a052da5b4b92 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Tue, 16 Nov 2021 14:21:50 -0600 Subject: [PATCH 10/39] Wrote its first fbh5 file! --- rawspec_file.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rawspec_file.c b/rawspec_file.c index 06ffd56..ceb7608 100644 --- a/rawspec_file.c +++ b/rawspec_file.c @@ -17,8 +17,13 @@ int open_output_file(callback_data_t *cb_data, const char * dest, const char *st int fd; const char * basename; char fname[PATH_MAX+1]; + char fileext[3]; // If dest is given and it's not empty + if(cb_data->flag_fbh5_output) + strcpy(fileext, ".h5"); + else + strcpy(fileext, ".fil"); if(dest && dest[0]) { // Look for last '/' in stem basename = strrchr(stem, '/'); @@ -29,9 +34,9 @@ int open_output_file(callback_data_t *cb_data, const char * dest, const char *st // If not found, use stem as basename basename = stem; } - snprintf(fname, PATH_MAX, "%s/%s.rawspec.%04d.fil", dest, basename, output_idx); + snprintf(fname, PATH_MAX, "%s/%s.rawspec.%04d.%s", dest, basename, output_idx, fileext); } else { - snprintf(fname, PATH_MAX, "%s.rawspec.%04d.fil", stem, output_idx); + snprintf(fname, PATH_MAX, "%s.rawspec.%04d.%s", stem, output_idx, fileext); } fname[PATH_MAX] = '\0'; if(cb_data->flag_fbh5_output) { @@ -42,6 +47,8 @@ int open_output_file(callback_data_t *cb_data, const char * dest, const char *st fbh5_open(&(cb_data->fbh5_ctx_ics), &(cb_data->fb_hdr), fname, cb_data->debug_callback); else fbh5_open(&(cb_data->fbh5_ctx_ant[antenna_index]), &(cb_data->fb_hdr), fname, cb_data->debug_callback); + if(cb_data->debug_callback) + printf("open_output_file: fbh5_open(%s) successful\n", fname); return ENABLER_FD_FOR_FBH5; } From b65a19f07159dfa4c7b5a44d0b487913754ad01e Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Tue, 16 Nov 2021 14:22:43 -0600 Subject: [PATCH 11/39] Wrote its first fbh5 file! --- rawspec.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/rawspec.c b/rawspec.c index 077f820..51d2bbc 100644 --- a/rawspec.c +++ b/rawspec.c @@ -60,14 +60,15 @@ ssize_t read_fully(int fd, void * buf, size_t bytes_to_read) } static struct option long_opts[] = { - {"fbh5", 0, NULL, '5'}, {"ant", 1, NULL, 'a'}, {"batch", 0, NULL, 'b'}, {"dest", 1, NULL, 'd'}, {"ffts", 1, NULL, 'f'}, {"gpu", 1, NULL, 'g'}, + {"help", 0, NULL, 'h'}, {"hdrs", 0, NULL, 'H'}, {"ICS", 1, NULL, 'I'}, + {"fbh5", 0, NULL, 'j'}, {"nchan", 1, NULL, 'n'}, {"outidx", 1, NULL, 'o'}, {"pols", 1, NULL, 'p'}, @@ -75,8 +76,8 @@ static struct option long_opts[] = { {"schan", 1, NULL, 's'}, {"splitant",0, NULL, 'S'}, {"ints", 1, NULL, 't'}, - {"help", 0, NULL, 'h'}, {"version", 0, NULL, 'v'}, + {"debug", 0, NULL, 'z'}, {0,0,0,0} }; @@ -91,7 +92,6 @@ void usage(const char *argv0) { "Usage: %s [options] STEM [...]\n" "\n" "Options:\n" - " -5, --fbh5 Format output Filterbank files as FBH5 (.h5) instead of SIGPROC(.fil)\n" " -a, --ant=ANT The 0-indexed antenna to exclusively process [-1]\n" " -b, --batch=BC Batch process BC coarse-channels at a time (1: auto, <1: disabled) [0]\n" " -d, --dest=DEST Destination directory or host:port\n" @@ -100,14 +100,16 @@ void usage(const char *argv0) { " -H, --hdrs Save headers to separate file\n" " -i, --ics=W1[,W2...] Output incoherent-sum (exclusively, unless with -S)\n" " specifying per antenna-weights or a singular, uniform weight\n" + " -j, --fbh5 Format output Filterbank files as FBH5 (.h5) instead of SIGPROC(.fil)\n" " -n, --nchan=N Number of coarse channels to process [all]\n" " -o, --outidx=N First index number for output files [0]\n" " -p --pols={1|4}[,...] Number of output polarizations [1]\n" " 1=total power, 4=cross pols, -4=full stokes\n" " -r, --rate=GBPS Desired net data rate in Gbps [6.0]\n" " -s, --schan=C First coarse channel to process [0]\n" - " -t, --ints=N1[,N2...] Spectra to integrate [51, 128, 3072]\n" " -S, --splitant Split output into per antenna files\n" + " -t, --ints=N1[,N2...] Spectra to integrate [51, 128, 3072]\n" + " -z, --debug Turn on selected debug output\n" "\n" " -h, --help Show this message\n" " -v, --version Show version and exit\n" @@ -194,6 +196,9 @@ char tmp[16]; int input_conjugated = -1; int only_output_ics = 0; + // Selected dynamic debugging + int flag_debugging = 0; + // FBH5 fields int flag_fbh5_output = 0; @@ -217,10 +222,14 @@ char tmp[16]; return 0; break; - case '5': // FBH5 output format requested + case 'j': // FBH5 output format requested flag_fbh5_output = 1; break; + case 'z': // Selected dynamic debugging + flag_debugging = 1; + break; + case 'a': // Antenna selection to process ant = strtol(optarg, NULL, 0); break; @@ -429,9 +438,11 @@ char tmp[16]; // Init user_data to be array of callback data structures ctx.user_data = &cb_data; - // Zero-out the callback data sructures + // Zero-out the callback data sructures. + // Turn on dynamic debugging if requested. for(i=0; idebug_callback) + printf("rawspec-main: open_output_file_per_antenna_and_write_header - successful\n"); } // Handle ICS. if(ctx.incoherently_sum) { @@ -810,6 +824,8 @@ char tmp[16]; // open any more output files, so print message and bail out. fprintf(stderr, "cannot open output file, giving up\n"); return 1; // Give up + if(cb_data->debug_callback) + printf("rawspec-main: open_output_file - successful\n"); } // Write filterbank header to SIGPROC output ICS file. From 2ebd4cdb8ecb4a6b814bbd82fa89f7b3f3853449 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 08:57:11 -0600 Subject: [PATCH 12/39] Fix command line options parsing --- rawspectest.c | 1153 ++++++------------------------------------------- 1 file changed, 121 insertions(+), 1032 deletions(-) diff --git a/rawspectest.c b/rawspectest.c index 51d2bbc..b12e46d 100644 --- a/rawspectest.c +++ b/rawspectest.c @@ -1,1083 +1,172 @@ -#define _GNU_SOURCE 1 - #include #include #include #include -#include #include -#include #include -#include -#include -#include -#include -#include -#include + +#include +#include #include "rawspec.h" -#include "rawspec_file.h" -#include "rawspec_socket.h" -#include "rawspec_version.h" -#include "rawspec_rawutils.h" -#include "rawspec_fbutils.h" -#include "fbh5_defs.h" #define ELAPSED_NS(start,stop) \ (((int64_t)stop.tv_sec-start.tv_sec)*1000*1000*1000+(stop.tv_nsec-start.tv_nsec)) -// This stringification trick is from "info cpp" -#define STRINGIFY1(s) #s -#define STRINGIFY(s) STRINGIFY1(s) - -#ifndef DEBUG_CALLBACKS -#define DEBUG_CALLBACKS (0) -#endif - -// Reads `bytes_to_read` bytes from `fd` into the buffer pointed to by `buf`. -// Returns the total bytes read or -1 on error. A non-negative return value -// will be less than `bytes_to_read` only of EOF is reached. -ssize_t read_fully(int fd, void * buf, size_t bytes_to_read) +void +dump_callback(rawspec_context * ctx, int output_product, int callback_type) { - ssize_t bytes_read; - ssize_t total_bytes_read = 0; - - while(bytes_to_read > 0) { - bytes_read = read(fd, buf, bytes_to_read); - if(bytes_read <= 0) { - if(bytes_read == 0) { - break; - } else { - return -1; - } - } - buf += bytes_read; - bytes_to_read -= bytes_read; - total_bytes_read += bytes_read; - } - - return total_bytes_read; + printf("cb %d\n", output_product); } -static struct option long_opts[] = { - {"ant", 1, NULL, 'a'}, - {"batch", 0, NULL, 'b'}, - {"dest", 1, NULL, 'd'}, - {"ffts", 1, NULL, 'f'}, - {"gpu", 1, NULL, 'g'}, - {"help", 0, NULL, 'h'}, - {"hdrs", 0, NULL, 'H'}, - {"ICS", 1, NULL, 'I'}, - {"fbh5", 0, NULL, 'j'}, - {"nchan", 1, NULL, 'n'}, - {"outidx", 1, NULL, 'o'}, - {"pols", 1, NULL, 'p'}, - {"rate", 1, NULL, 'r'}, - {"schan", 1, NULL, 's'}, - {"splitant",0, NULL, 'S'}, - {"ints", 1, NULL, 't'}, - {"version", 0, NULL, 'v'}, - {"debug", 0, NULL, 'z'}, - {0,0,0,0} -}; - -void usage(const char *argv0) { - const char * bname = basename(argv0); - // Should "never" happen - if(!bname) { - bname = argv0; - } - - fprintf(stderr, - "Usage: %s [options] STEM [...]\n" - "\n" - "Options:\n" - " -a, --ant=ANT The 0-indexed antenna to exclusively process [-1]\n" - " -b, --batch=BC Batch process BC coarse-channels at a time (1: auto, <1: disabled) [0]\n" - " -d, --dest=DEST Destination directory or host:port\n" - " -f, --ffts=N1[,N2...] FFT lengths [1048576, 8, 1024]\n" - " -g, --GPU=IDX Select GPU device to use [0]\n" - " -H, --hdrs Save headers to separate file\n" - " -i, --ics=W1[,W2...] Output incoherent-sum (exclusively, unless with -S)\n" - " specifying per antenna-weights or a singular, uniform weight\n" - " -j, --fbh5 Format output Filterbank files as FBH5 (.h5) instead of SIGPROC(.fil)\n" - " -n, --nchan=N Number of coarse channels to process [all]\n" - " -o, --outidx=N First index number for output files [0]\n" - " -p --pols={1|4}[,...] Number of output polarizations [1]\n" - " 1=total power, 4=cross pols, -4=full stokes\n" - " -r, --rate=GBPS Desired net data rate in Gbps [6.0]\n" - " -s, --schan=C First coarse channel to process [0]\n" - " -S, --splitant Split output into per antenna files\n" - " -t, --ints=N1[,N2...] Spectra to integrate [51, 128, 3072]\n" - " -z, --debug Turn on selected debug output\n" - "\n" - " -h, --help Show this message\n" - " -v, --version Show version and exit\n" - , bname - ); +void show_gpu_memory_usage(int gpu_id) { + size_t in_use, available, total; + int gpu_id_selected; + cudaSetDevice(gpu_id); + cudaGetDevice(&gpu_id_selected); + cudaMemGetInfo(&available, &total); + in_use = total - available; + printf("GPU %d memory, in use: %ld MiB, free: %ld MiB, total: %ld MiB\n", + gpu_id_selected, + (long) ((float) in_use / 1.0e6), + (long) ((float) available / 1.0e6), + (long) ((float) total / 1.0e6)); } -int open_headers_file(const char * dest, const char *stem) +int main(int argc, char * argv[]) { - int fd; - const char * basename; - char fname[PATH_MAX+1]; - - // If dest is given and it's not empty - if(dest && dest[0]) { - // Look for last '/' in stem - basename = strrchr(stem, '/'); - if(basename) { - // If found, advance beyond it to first char of basename - basename++; - } else { - // If not found, use stem as basename - basename = stem; - } - snprintf(fname, PATH_MAX, "%s/%s.rawspec.headers", dest, basename); - } else { - snprintf(fname, PATH_MAX, "%s.rawspec.headers", stem); - } - fname[PATH_MAX] = '\0'; - fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC, 0664); - if(fd == -1) { - perror(fname); - } else { - posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); - } - return fd; -} - -int main(int argc, char *argv[]) -{ - int si; // Indexes the stems - int fi; // Indexes the files for a given stem - int bi; // Counts the blocks processed for a given file int i; -int j, k; -char tmp[16]; - void * pv; - int fdin; - int fdhdrs = -1; - int next_stem = 0; - int save_headers = 0; - int per_ant_out = 0; - unsigned int Nc; // Number of coarse channels across the observation (possibly multi-antenna) - unsigned int Ncpa;// Number of coarse channels per antenna - unsigned int Np; // Number of polarizations - unsigned int Ntpb; // Number of time samples per block - unsigned int Nbps; // Number of bits per sample - uint64_t block_byte_length; // Compute the length once - char expand4bps_to8bps; // Expansion flag - int64_t pktidx0; - int64_t pktidx; - int64_t dpktidx; - char fname[PATH_MAX+1]; - int opt; - char * argv0; - char * pchar; - char * bfname; - char * dest = NULL; // default output dest is same place as input stem - char * ics_output_stem = NULL; - rawspec_output_mode_t output_mode = RAWSPEC_FILE; - char * dest_port = NULL; // dest port for network output - int fdout; - int open_flags; - size_t bytes_read; - size_t total_bytes_read; - off_t pos; - rawspec_raw_hdr_t raw_hdr; - callback_data_t cb_data[MAX_OUTPUTS]; - rawspec_context ctx; - int ant = -1; - unsigned int schan = 0; - unsigned int nchan = 0; - unsigned int outidx = 0; - int input_conjugated = -1; - int only_output_ics = 0; - - // Selected dynamic debugging - int flag_debugging = 0; - - // FBH5 fields - int flag_fbh5_output = 0; - - // For net data rate rate calculations - double rate = 6.0; - double sum_inv_na; - uint64_t total_packets = 0; - uint64_t total_bytes = 0; - uint64_t total_ns = 0; - - // Init rawspec context - memset(&ctx, 0, sizeof(ctx)); - ctx.Npolout[0] = 1; // others will be set later - - // Parse command line. - argv0 = argv[0]; - while((opt=getopt_long(argc,argv,"a:b:d:f:g:HI:i:n:o:p:r:Ss:t:hv",long_opts,NULL))!=-1) { - switch (opt) { - case 'h': // Help - usage(argv0); - return 0; - break; - - case 'j': // FBH5 output format requested - flag_fbh5_output = 1; - break; - - case 'z': // Selected dynamic debugging - flag_debugging = 1; - break; - - case 'a': // Antenna selection to process - ant = strtol(optarg, NULL, 0); - break; - - case 'b': // Batch-channels - ctx.Nbc = strtol(optarg, NULL, 0); - break; - - case 'd': // Output destination - dest = optarg; - // If dest contains at least one ':', it's HOST:PORT and we're - // outputting over the network. - pchar = strrchr(dest, ':'); - if(pchar) { - // NUL terminate hostname, advance to port - *pchar++ = '\0'; - dest_port = pchar; - output_mode = RAWSPEC_NET; - } - break; - - case 'f': // Fine channel(s) per coarse channel - for(i=0, pchar = strtok(optarg,","); - pchar != NULL; i++, pchar = strtok(NULL, ",")) { - if(i>=MAX_OUTPUTS){ - fprintf(stderr, - "error: up to %d fine channel counts supported.\n", MAX_OUTPUTS); - return 1; - } - ctx.Nts[i] = strtoul(pchar, NULL, 0); - } - // If no comma (i.e. single value) - if(i==0) { - ctx.Nts[0] = strtoul(optarg, NULL, 0); - } - break; - - case 'g': // GPU device to use - ctx.gpu_index = strtol(optarg, NULL, 0); - printf("using requested GPU: %d\n", ctx.gpu_index); - break; - - case 'H': // Save headers - save_headers = 1; - break; + int j; + rawspec_context ctx = {0}; - case 'i': // Incoherently sum exclusively - only_output_ics = 1; - ctx.incoherently_sum = 1; - ctx.Naws = 1; - // Count number of - for(i=0; i < strlen(optarg); i++) - ctx.Naws += optarg[i]==','; - - char *weight_end; - ctx.Aws = malloc(ctx.Naws*sizeof(float)); + // Timing variables + struct timespec ts_start, ts_stop; + uint64_t elapsed_ns=0; - for(i=0; i < ctx.Naws; i++){ - ctx.Aws[i] = strtof(optarg, &weight_end); - optarg = weight_end; - } - - break; + int blocsize = 92274688; + size_t nfine; - case 'n': // Number of coarse channels to process - nchan = strtoul(optarg, NULL, 0); - break; - - case 'o': // Index number for first output product file name - outidx = strtoul(optarg, NULL, 0); - break; - - case 'p': // Number of pol products to output - for(i=0, pchar = strtok(optarg,","); - pchar != NULL; i++, pchar = strtok(NULL, ",")) { - if(i>=MAX_OUTPUTS){ - fprintf(stderr, - "error: up to %d pol modes supported.\n", MAX_OUTPUTS); - return 1; - } - ctx.Npolout[i] = strtoul(pchar, NULL, 0); - } - // If no comma (i.e. single value) - if(i==0) { - ctx.Npolout[0] = strtoul(optarg, NULL, 0); - } - break; - - case 'r': // Relative rate to send packets - rate = strtod(optarg, NULL); - break; - - case 's': // First coarse channel to process - schan = strtoul(optarg, NULL, 0); - break; - - case 'S': // Split output per antenna - per_ant_out = 1; - break; - - case 't': // Number of spectra to accumumate - for(i=0, pchar = strtok(optarg,","); - pchar != NULL; i++, pchar = strtok(NULL, ",")) { - if(i>=MAX_OUTPUTS){ - fprintf(stderr, - "error: up to %d integration counts supported.\n", MAX_OUTPUTS); - return 1; - } - ctx.Nas[i] = strtoul(pchar, NULL, 0); - } - // If no comma (i.e. single value) - if(i==0) { - ctx.Nas[0] = strtoul(optarg, NULL, 0); - } - break; - - case 'v': // Version - printf("rawspec %s\n", STRINGIFY(RAWSPEC_VERSION)); - printf("librawspec %s\n", rawspec_version_string()); - return 0; - break; - - case '?': // Command line parsing error - default: - printf("Unknown CLI option '%c'\n", opt); - usage(argv0); - return 1; - break; + // Show librawspec version on startup + printf("using librawspec %s\n", rawspec_version_string()); + + ctx.No = 4; + ctx.Np = 2; + ctx.Nc = 88; + ctx.Nbc = 1; + ctx.Nbps = 8; + if(argc > 1) { + ctx.Nbps = strtoul(argv[1], NULL, 0); + if(ctx.Nbps == 0) { + ctx.Nbps = 8; } } - - // Skip past option args - argc -= optind; - argv += optind; - - // If no stems given, print usage and exit - if(argc == 0) { - usage(argv0); - return 1; - } - - // Show librawspec version on startup - printf("rawspec using librawspec %s\n", rawspec_version_string()); - - // If schan is non-zero, nchan must be too - if(schan != 0 && nchan == 0) { - fprintf(stderr, "error: nchan must be non-zero if schan is non-zero\n"); + printf("using %u bits per sample\n", ctx.Nbps); + ctx.Ntpb = blocsize / (2 * ctx.Np * ctx.Nc * ctx.Nbps/8); + ctx.Nts[0] = (1<<20); + ctx.Nts[1] = (1<<3); + ctx.Nts[2] = (1<<10); + ctx.Nts[3] = (1<<10); + // One dump per output product + ctx.Nas[0] = (1<<(20 - 20)); + ctx.Nas[1] = (1<<(20 - 3)); + ctx.Nas[2] = (1<<(20 - 10)); + ctx.Nas[3] = (1<<(20 - 10)); + // Auto-calculate Nb/Nb_host and let library manage input block buffers + ctx.Nb = 0; + ctx.Nb_host = 0; + ctx.h_blkbufs = NULL; + // Dump callback + ctx.dump_callback = dump_callback; + // Pol modes for each output product + ctx.Npolout[0] = 1; + ctx.Npolout[1] = 1; + ctx.Npolout[2] = 4; + ctx.Npolout[3] = -4; + + ctx.gpu_index = 0; + ctx.input_conjugated = 1; + + // Initialize + if(rawspec_initialize(&ctx)) { + fprintf(stderr, "initialization failed\n"); return 1; } + printf("initialization succeeded, RAWSPEC_BLOCSIZE=%u\n", + RAWSPEC_BLOCSIZE(&ctx)); - // Saving headers is only supported for file output - if(save_headers && output_mode != RAWSPEC_FILE) { - fprintf(stderr, - "warning: saving headers is only supported for file output\n"); - save_headers = 0; + // Setup input data + for(i=0; i 0) { - // Copy value from previous output product - ctx.Npolout[i] = ctx.Npolout[i-1]; - } else if(ctx.Npolout[i]!=1 && abs(ctx.Npolout[i])!=4) { - fprintf(stderr, - "error: number of output pols must be 1 or +/- 4\n"); - return 1; - } - - // Full-pol mode is not supported for network output - if(ctx.Npolout[i] != 1 && output_mode != RAWSPEC_FILE) { - fprintf(stderr, - "error: full-pol mode is not supported for network output\n"); - return 1; - } + memset(ctx.h_pwrbuf[i], 0x55, ctx.h_pwrbuf_size[i]); } - // Init user_data to be array of callback data structures - ctx.user_data = &cb_data; + for(i=0; i<4; i++) { + clock_gettime(CLOCK_MONOTONIC, &ts_start); - // Zero-out the callback data sructures. - // Turn on dynamic debugging if requested. - for(i=0; i 1 && !(per_ant_out || ctx.incoherently_sum)){ - printf("NANTS = %d >1: Enabling --split-ant in lieu of neither --split-ant nor --ics flags.\n", raw_hdr.nants); - per_ant_out = 1; - } - - // If splitting output per antenna, re-alloc the fd array. - if(per_ant_out) { - if(output_mode == RAWSPEC_FILE){ - if(ant != -1){ - printf("Ignoring --ant %d option:\n\t", ant); - } - printf("Splitting output per %d antennas\n", - raw_hdr.nants); - // close previous - for(i=0; i raw_hdr.nants - 1 || ant < 0) { - printf("bad antenna selection: ant <> {0, nants} (%u <> {0, %d})\n", - ant, raw_hdr.nants); - close(fdin); - break; // Goto next stem - } - if(schan >= Ncpa) { - printf("bad schan specification with antenna selection: " - "schan > antnchan {obsnchan/nants} (%u > %u {%d/%d})\n", - schan, Ncpa, raw_hdr.obsnchan, raw_hdr.nants); - close(fdin); - break; // Goto next stem - } - - // Set Nc to Ncpa and skip previous antennas - printf("Selection of antenna %d equates to a starting channel of %d\n", ant, ant*Ncpa); - schan += ant * Ncpa; - Nc = Ncpa; - } - - // If processing a subset of coarse channels - if(nchan != 0) { - // Validate schan and nchan - if(ant == -1 && // no antenna selection - (schan + nchan > Nc)) { - - printf("bad channel range: schan + nchan > obsnchan (%u + %u > %d)\n", - schan, nchan, raw_hdr.obsnchan); - close(fdin); - break; // Goto next stem - } - else if(ant != -1 && // antenna selection - (schan + nchan > (ant + 1) * Ncpa)) { - printf("bad channel range: schan + nchan > antnchan {obsnchan/nants} (%u + %u > %d {%d/%d})\n", - schan - ant * Ncpa, nchan, Ncpa, raw_hdr.obsnchan, raw_hdr.nants); - close(fdin); - break; // Goto next stem - } - // Use nchan as Nc - Nc = nchan; - } - - // Determine if input is conjugated - input_conjugated = (raw_hdr.obsbw < 0) ? 1 : 0; - - // If block dimensions or input conjugation have changed - if(Nc != ctx.Nc || Np != ctx.Np || Nbps != ctx.Nbps || Ntpb != ctx.Ntpb - || input_conjugated != ctx.input_conjugated) { - // Cleanup previous block, if it has been initialized - if(ctx.Ntpb != 0) { - rawspec_cleanup(&ctx); - } - // Remember new dimensions and input conjugation - ctx.Nant = raw_hdr.nants; - ctx.Nc = Nc; - ctx.Np = Np; - ctx.Ntpb = Ntpb; - ctx.Nbps = Nbps; - ctx.input_conjugated = input_conjugated; - - // Initialize for new dimensions and/or conjugation - ctx.Nb = 0; // auto-calculate - ctx.Nb_host = 0; // auto-calculate - ctx.h_blkbufs = NULL; // auto-allocate - if(rawspec_initialize(&ctx)) { - fprintf(stderr, "rawspec initialization failed\n"); - close(fdin); - // Forget new dimensions - ctx.Nant = 0; - ctx.Nc = 0; - ctx.Np = 0; - ctx.Ntpb = 0; - ctx.Nbps = 0; - return 1; // fixes issue #23 - } else { - // printf("initialization succeeded for new block dimensions\n"); - block_byte_length = (2 * ctx.Np * ctx.Nc * ctx.Nbps)/8 * ctx.Ntpb; - - // The GPU supports only 8bit and 16bit sample bit-widths. The strategy - // for handling 4bit samples is to expand them out to 8bits, and there-onwards - // use the expanded 8bit samples. The device side rawspec_initialize actually still - // complains about the indication of the samples being 4bits. But the - // expand4bps_to8bps flag is used to call rawspec_copy_blocks_to_gpu_expanding_complex4, - // leading to the samples being expanded before any device side computation happens - // in rawspec_start_processing. The ctx.Nbps is left as 8. - if (ctx.Nbps == 8 && Nbps == 4){ - printf("CUDA memory initialised for %d bits per sample,\n\t" - "will expand header specified %d bits per sample.\n", ctx.Nbps, Nbps); - expand4bps_to8bps = 1; - } - - // Copy fields from ctx to cb_data - for(i=0; idebug_callback) - printf("rawspec-main: open_output_file_per_antenna_and_write_header - successful\n"); - } - // Handle ICS. - if(ctx.incoherently_sum) { - cb_data[i].fd_ics = open_output_file(&cb_data[i], - dest, - ics_output_stem, - outidx + i, - /* ICS */ -1); - if(cb_data[i].fd_ics == -1) { - // If we can't open this output file, we probably won't be able to - // open any more output files, so print message and bail out. - fprintf(stderr, "cannot open output file, giving up\n"); - return 1; // Give up - if(cb_data->debug_callback) - printf("rawspec-main: open_output_file - successful\n"); - } - - // Write filterbank header to SIGPROC output ICS file. - // If FBH5, the header was already written by fbh5_open(). - if(! flag_fbh5_output) { - fb_fd_write_header(cb_data[i].fd_ics, &cb_data[i].fb_hdr); - } - } // if(ctx.incoherently_sum) - } // if(output_mode == RAWSPEC_FILE) - } // for(i=0; i pktidx) { - dpktidx = raw_hdr.pktidx - pktidx; - } - - // Handle cases were the current pktidx is not the expected distance - // from the previous pktidx. - if(raw_hdr.pktidx - pktidx != dpktidx) { - // Cannot go backwards or forwards by non-multiple of dpktidx - if(raw_hdr.pktidx < pktidx) { - printf("got backwards jump in pktidx: %ld -> %ld\n", - pktidx, raw_hdr.pktidx); - // Give up on this stem and go to next stem - next_stem = 1; - break; - } else if((raw_hdr.pktidx - pktidx) % dpktidx != 0) { - printf("got misaligned jump in pktidx: (%ld - %ld) %% %ld != 0\n", - raw_hdr.pktidx, pktidx, dpktidx); - // Give up on this stem and go to next stem - next_stem = 1; - break; - } else if (raw_hdr.pktidx == pktidx ){ - printf("got null jump in pktidx: (%ld - %ld) == 0\n", - raw_hdr.pktidx, pktidx); - // just skip this block - break; - } - - // Put in filler blocks of zeros - while(raw_hdr.pktidx - pktidx != dpktidx) { - // Increment pktidx to next missing value - pktidx += dpktidx; + rawspec_start_processing(&ctx, -1); + rawspec_wait_for_completion(&ctx); - // Fill block buffer with zeros - memset(ctx.h_blkbufs[bi%ctx.Nb_host], 0, raw_hdr.blocsize); + clock_gettime(CLOCK_MONOTONIC, &ts_stop); -#ifdef VERBOSE - fprintf(stderr, "%3d %016lx:", bi, pktidx); - fprintf(stderr, " -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --\n"); -#endif // VERBOSE - - // If this is the last block of an input buffer, start processing - if(bi % ctx.Nb == ctx.Nb - 1) { -#ifdef VERBOSE - fprintf(stderr, "block %3d buf 0: ", bi); - for(j=0; j<16; j++) { - fprintf(stderr, " %02x", ctx.h_blkbufs[0][j] & 0xff); - } - fprintf(stderr, "\n"); -#endif // VERBOSE - rawspec_wait_for_completion(&ctx); - rawspec_copy_blocks_to_gpu_and_start_processing(&ctx, ctx.Nb, expand4bps_to8bps, RAWSPEC_FORWARD_FFT); - } - - // Increment block counter - bi++; - } // filler zero blocks - } // irregular pktidx step - - // Seek past first schan channel - lseek(fdin, (2 * ctx.Np * schan * Nbps)/8 * ctx.Ntpb, SEEK_CUR); - - // Read ctx.Nc coarse channels from this block - bytes_read = read_fully(fdin, - ctx.h_blkbufs[bi % ctx.Nb_host], - (expand4bps_to8bps ? block_byte_length/2 : block_byte_length)); - - // Seek past channels after schan+nchan - lseek(fdin, (2 * ctx.Np * (raw_hdr.obsnchan-(schan+Nc)) * Nbps)/8 * ctx.Ntpb, SEEK_CUR); - - if(bytes_read == -1) { - perror("read"); - next_stem = 1; - break; // Goto next file - } else if(bytes_read < (expand4bps_to8bps ? block_byte_length/2 : block_byte_length)) { - fprintf(stderr, "incomplete block at EOF\n"); - next_stem = 1; - break; // Goto next file - } - total_bytes_read += bytes_read; - -#ifdef VERBOSE - fprintf(stderr, "%3d %016lx:", bi, raw_hdr.pktidx); - for(j=0; j<16; j++) { - fprintf(stderr, " %02x", ctx.h_blkbufs[bi%ctx.Nb_host][j] & 0xff); - } - fprintf(stderr, "\n"); -#endif // VERBOSE - - // If this is the last block of an input buffer, start processing - if(bi % ctx.Nb == ctx.Nb - 1) { -#ifdef VERBOSE - fprintf(stderr, "block %3d buf 0: ", bi); - for(j=0; j<16; j++) { - fprintf(stderr, " %02x", ctx.h_blkbufs[0][j] & 0xff); - } - fprintf(stderr, "\n"); -#endif // VERBOSE - rawspec_wait_for_completion(&ctx); - rawspec_copy_blocks_to_gpu_and_start_processing(&ctx, ctx.Nb, expand4bps_to8bps, RAWSPEC_FORWARD_FFT); - } - - // Remember pktidx - pktidx = raw_hdr.pktidx; - - // Increment block index to next block (which may be in the next file) - bi++; - - // Read obs params of next block - pos = rawspec_raw_read_header(fdin, &raw_hdr); - if(pos <= 0) { - if(pos == -1) { - fprintf(stderr, "error getting obs params from %s [%s]\n", - fname, strerror(errno)); - } - break; - } - } // For each block - - // Done with input file - close(fdin); - - // If skipping to next stem - if(next_stem) { - next_stem = 0; - // break out of each file loop - break; - } - } // each file for stem - - // Wait for GPU work to complete - if(ctx.Nc) { - rawspec_wait_for_completion(&ctx); - } - - // Close output files - if(output_mode == RAWSPEC_FILE) { - for(i=0; i 0) { - printf(" (%u packets, %.3f Gbps)", cb_data[i].total_packets, - 8.0 * cb_data[i].total_bytes / cb_data[i].total_ns); - - total_packets += cb_data[i].total_packets; - total_bytes += cb_data[i].total_bytes; - total_ns += cb_data[i].total_ns; + nfine = ctx.Nc * ctx.Nts[i]; + for(j=0; j<16; j++) { + if(ctx.Npolout[i] == 1) { + printf("output product %d chan %d %f\n", i, j, ctx.h_pwrbuf[i][j]); + } else { + printf("output product %d chan %d %f %f %f %f\n", i, j, + ctx.h_pwrbuf[i][ j], + ctx.h_pwrbuf[i][1*nfine+j], + ctx.h_pwrbuf[i][2*nfine+j], + ctx.h_pwrbuf[i][3*nfine+j]); + } } - printf("\n"); } - if(total_ns > 0) { - printf("combined total : %lu packets, %.3f Gbps\n", - total_packets, 8.0 * total_bytes / total_ns); - } + // For checking mempry usage + //printf("sleeping for 10 seconds..."); + printf("\nchecking GPU memory usage ...\n"); + fflush(stdout); + //sleep(10); + show_gpu_memory_usage(ctx.gpu_index); + printf("done\n"); + + printf("cleaning up..."); + fflush(stdout); + rawspec_cleanup(&ctx); + printf("done\n"); return 0; } - - From ad676824c3eb1c2319f3533aeaaee34cc158c28e Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 08:57:50 -0600 Subject: [PATCH 13/39] Set deflation level to 1 --- fbh5_open.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fbh5_open.c b/fbh5_open.c index 01b5ea2..483e2bc 100644 --- a/fbh5_open.c +++ b/fbh5_open.c @@ -23,7 +23,7 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p char wstr[256]; // sprintf target unsigned hdf5_majnum, hdf5_minnum, hdf5_relnum; // Version/release info for the HDF5 library - int deflate_level = 6; // 0=no deflation, 9=highest, 6=recommended + int deflate_level = 1; // 0=no deflation, 9=highest, 6=recommended H5get_libversion(&hdf5_majnum, &hdf5_minnum, &hdf5_relnum); if(debug_callback) { From 709a5e0ed512211f70eac5f50508e70334000040 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 09:27:44 -0600 Subject: [PATCH 14/39] Update rawspec_file.c --- rawspec_file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rawspec_file.c b/rawspec_file.c index ceb7608..9f7ab7f 100644 --- a/rawspec_file.c +++ b/rawspec_file.c @@ -21,9 +21,9 @@ int open_output_file(callback_data_t *cb_data, const char * dest, const char *st // If dest is given and it's not empty if(cb_data->flag_fbh5_output) - strcpy(fileext, ".h5"); + strcpy(fileext, "h5"); else - strcpy(fileext, ".fil"); + strcpy(fileext, "fil"); if(dest && dest[0]) { // Look for last '/' in stem basename = strrchr(stem, '/'); From 6d2c1b08ad58c32481e03fcb00b437940a68c404 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 10:14:16 -0600 Subject: [PATCH 15/39] while((opt=getopt_long(argc,argv,"a:b:d:f:g:HI:i:n:o:p:r:Ss:t:hv:j:z:",long_opts,NULL))!=-1) --- rawspec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rawspec.c b/rawspec.c index 51d2bbc..f2d004c 100644 --- a/rawspec.c +++ b/rawspec.c @@ -215,7 +215,7 @@ char tmp[16]; // Parse command line. argv0 = argv[0]; - while((opt=getopt_long(argc,argv,"a:b:d:f:g:HI:i:n:o:p:r:Ss:t:hv",long_opts,NULL))!=-1) { + while((opt=getopt_long(argc,argv,"a:b:d:f:g:HI:i:n:o:p:r:Ss:t:hv:j:z:",long_opts,NULL))!=-1) { switch (opt) { case 'h': // Help usage(argv0); From 817f05ed82a69c248f97a86b3e0a100e39a4c8b5 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 10:26:49 -0600 Subject: [PATCH 16/39] while((opt=getopt_long(argc,argv,"a:b:d:f:g:HjzI:i:n:o:p:r:Ss:t:hv",long_opts,NULL))!=-1) --- rawspec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rawspec.c b/rawspec.c index f2d004c..6483464 100644 --- a/rawspec.c +++ b/rawspec.c @@ -215,7 +215,7 @@ char tmp[16]; // Parse command line. argv0 = argv[0]; - while((opt=getopt_long(argc,argv,"a:b:d:f:g:HI:i:n:o:p:r:Ss:t:hv:j:z:",long_opts,NULL))!=-1) { + while((opt=getopt_long(argc,argv,"a:b:d:f:g:HjzI:i:n:o:p:r:Ss:t:hv",long_opts,NULL))!=-1) { switch (opt) { case 'h': // Help usage(argv0); From b946d2b5e3091b40f245b9fe9ba98d85984cdfbb Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 13:12:50 -0600 Subject: [PATCH 17/39] add debugging for fbh5_ctx-empty bug --- rawspec_file.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rawspec_file.c b/rawspec_file.c index 9f7ab7f..2c1d6d9 100644 --- a/rawspec_file.c +++ b/rawspec_file.c @@ -109,6 +109,8 @@ void * dump_file_thread_func(void *arg) // Assume that the following file-descriptors aren't valid break; } + if(cb_data->debug_callback) + printf("dump_file_thread_func: write for antenna %ld-%ld-%ld\n", k, j, i); if(cb_data->flag_fbh5_output) { fbh5_write(&(cb_data->fbh5_ctx_ant[i]), &(cb_data->fb_hdr), @@ -125,6 +127,8 @@ void * dump_file_thread_func(void *arg) } // for(size_t k = 0; k < cb_data->Nds; k++) } // if(cb_data->per_ant_out) else { // nants = 0; single output file + if(cb_data->debug_callback) + printf("dump_file_thread_func: write for nants=0\n"); if(cb_data->flag_fbh5_output) { fbh5_write(&(cb_data->fbh5_ctx_ant[0]), &(cb_data->fb_hdr), @@ -140,6 +144,8 @@ void * dump_file_thread_func(void *arg) } // if(cb_data->fd && cb_data->h_pwrbuf) if(cb_data->fd_ics && cb_data->h_icsbuf) { + if(cb_data->debug_callback) + printf("dump_file_thread_func: write for ICS\n"); if(cb_data->flag_fbh5_output) { fbh5_write(&(cb_data->fbh5_ctx_ics), &(cb_data->fb_hdr), From 3f73103b3ff0194138ba8984301954bc9659f60d Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 13:13:35 -0600 Subject: [PATCH 18/39] add debugging for fbh5_ctx-empty bug --- fbh5_write.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fbh5_write.c b/fbh5_write.c index a9a5620..17b6f00 100644 --- a/fbh5_write.c +++ b/fbh5_write.c @@ -27,6 +27,8 @@ void fbh5_write(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, void * p_buffe /* * Initialise write loop. */ + if(debug_callback) + fbh5_show_context("fbh5_write", p_fbh5_ctx); ntints = bufsize / p_fbh5_ctx->tint_size; // Compute the number of time integrations in the current dump. p_fbh5_ctx->dump_count += 1; // Bump the dump count. bufptr = p_buffer; // Point to the first time integration in the dump. From a42d65f139f326ce6a3527f04a7a8d404917d598 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 13:14:18 -0600 Subject: [PATCH 19/39] add debugging for fbh5_ctx-empty bug --- fbh5_util.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/fbh5_util.c b/fbh5_util.c index 821d17d..7d51914 100644 --- a/fbh5_util.c +++ b/fbh5_util.c @@ -134,3 +134,22 @@ void fbh5_set_ds_label(fbh5_context_t * p_fbh5_ctx, char * label, int dims_index } } + +/*** + Display some of the fbh5_context values. +***/ +void fbh5_show_context(char * caller, fbh5_context_t * p_fbh5_ctx) { + if(p_fbh5_ctx == NULL) { + printf("*** fbh5_show_context: p_fbh5_ctx = NULL !!!"); + return; + } + printf("fbh5_show_context(%s): active = %d\n", caller, p_fbh5_ctx->active); + printf("fbh5_show_context(%s): elem_size = %d\n", caller, p_fbh5_ctx->elem_size); + printf("fbh5_show_context(%s): tint_size = %ld\n", caller, p_fbh5_ctx->tint_size); + printf("fbh5_show_context(%s): offset_dims = (%lld, %lld, %lld)\n", + caller, p_fbh5_ctx->offset_dims[0], p_fbh5_ctx->offset_dims[1], p_fbh5_ctx->offset_dims[2]); + printf("fbh5_show_context(%s): filesz_dims = (%lld, %lld, %lld)\n", + caller, p_fbh5_ctx->filesz_dims[0], p_fbh5_ctx->filesz_dims[1], p_fbh5_ctx->filesz_dims[2]); + printf("fbh5_show_context(%s): byte_count = %ld\n", caller, p_fbh5_ctx->byte_count); + printf("fbh5_show_context(%s): dump_count = %ld\n", caller, p_fbh5_ctx->dump_count); +} From ef387ab356d07f377e33e657df693797f5fcc43c Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 13:15:02 -0600 Subject: [PATCH 20/39] add debugging for fbh5_ctx-empty bug --- fbh5_defs.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/fbh5_defs.h b/fbh5_defs.h index efb28e7..d8434cb 100644 --- a/fbh5_defs.h +++ b/fbh5_defs.h @@ -54,11 +54,7 @@ void fbh5_set_dataset_double_attr(hid_t dataset_id, char * tag, double * p_va void fbh5_set_dataset_int_attr(hid_t dataset_id, char * tag, int * p_value, int debug_callback); void fbh5_write_metadata(hid_t dataset_id, fb_hdr_t * p_metadata, int debug_callback); void fbh5_set_ds_label(fbh5_context_t * p_fbh5_ctx, char * label, int dims_index, int debug_callback); - -/* - * Debugging aid - */ -void hexdump(const char *label, const void *thingaddr, const int thingsz, int per_line); +void fbh5_show_context(char * caller, fbh5_context_t * p_fbh5_ctx); #endif From 56304c15b04966ef3aeafd28cf5c59305e096010 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 13:16:04 -0600 Subject: [PATCH 21/39] add debugging for fbh5_ctx-empty bug --- fbh5_open.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fbh5_open.c b/fbh5_open.c index 483e2bc..1261770 100644 --- a/fbh5_open.c +++ b/fbh5_open.c @@ -63,10 +63,6 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p p_fbh5_ctx->offset_dims[0] = 0; p_fbh5_ctx->offset_dims[1] = 0; p_fbh5_ctx->offset_dims[2] = 0; - if(debug_callback) { - printf("fbh5_open: Data element byte size = %d.\n", p_fbh5_ctx->elem_size); - printf("fbh5_open: Time integration byte size = %ld.\n", p_fbh5_ctx->tint_size); - } /* * Open HDF5 file. Overwrite it if preexisting. @@ -198,6 +194,7 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p * Bye-bye. */ p_fbh5_ctx->active = 1; + if(debug_callback) + fbh5_show_context("fbh5_open", p_fbh5_ctx); } - From 12de9d8a2e85c1aca36c3e71eb8c28a485250a70 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 13:17:44 -0600 Subject: [PATCH 22/39] renove option -I (capital I) --- rawspec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rawspec.c b/rawspec.c index 6483464..f427350 100644 --- a/rawspec.c +++ b/rawspec.c @@ -215,7 +215,7 @@ char tmp[16]; // Parse command line. argv0 = argv[0]; - while((opt=getopt_long(argc,argv,"a:b:d:f:g:HjzI:i:n:o:p:r:Ss:t:hv",long_opts,NULL))!=-1) { + while((opt=getopt_long(argc,argv,"a:b:d:f:g:Hjz:i:n:o:p:r:Ss:t:hv",long_opts,NULL))!=-1) { switch (opt) { case 'h': // Help usage(argv0); From 077ac7dc4c121db941a36eb904a019e19fbb896e Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 17 Nov 2021 14:44:57 -0600 Subject: [PATCH 23/39] Fix issue #36 The previous logic was causing fbh5_write() to crash because it was passed an undefined FBH5 context, hence a divide by zero. --- rawspec_file.c | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/rawspec_file.c b/rawspec_file.c index 2c1d6d9..d9da1bd 100644 --- a/rawspec_file.c +++ b/rawspec_file.c @@ -96,6 +96,24 @@ void * dump_file_thread_func(void *arg) { callback_data_t * cb_data = (callback_data_t *)arg; + // Single antenna case + if(cb_data->fd && cb_data->h_pwrbuf && (cb_data->Nant == 1)) { + if(cb_data->debug_callback) + printf("dump_file_thread_func: write for nants=0\n"); + if(cb_data->flag_fbh5_output) { + fbh5_write(&(cb_data->fbh5_ctx_ant[0]), + &(cb_data->fb_hdr), + cb_data->h_pwrbuf, + cb_data->h_pwrbuf_size, + cb_data->debug_callback); + } else { + write(cb_data->fd[0], + cb_data->h_pwrbuf, + cb_data->h_pwrbuf_size); + } // if(cb_data->flag_fbh5_output) + } + + // Multiple antennas, split output if(cb_data->fd && cb_data->h_pwrbuf) { if(cb_data->per_ant_out) { size_t spectra_stride = cb_data->h_pwrbuf_size / (cb_data->Nds * sizeof(float)); @@ -126,23 +144,9 @@ void * dump_file_thread_func(void *arg) } // for(size_t j = 0; j < cb_data->fb_hdr.nifs; j++) } // for(size_t k = 0; k < cb_data->Nds; k++) } // if(cb_data->per_ant_out) - else { // nants = 0; single output file - if(cb_data->debug_callback) - printf("dump_file_thread_func: write for nants=0\n"); - if(cb_data->flag_fbh5_output) { - fbh5_write(&(cb_data->fbh5_ctx_ant[0]), - &(cb_data->fb_hdr), - cb_data->h_pwrbuf, - cb_data->h_pwrbuf_size, - cb_data->debug_callback); - } else { - write(cb_data->fd[0], - cb_data->h_pwrbuf, - cb_data->h_pwrbuf_size); - } // if(cb_data->flag_fbh5_output) - } // if(cb_data->per_ant_out) ... else } // if(cb_data->fd && cb_data->h_pwrbuf) - + + // Multiple antennas, ICE output if(cb_data->fd_ics && cb_data->h_icsbuf) { if(cb_data->debug_callback) printf("dump_file_thread_func: write for ICS\n"); From d3aef1e6d9e63b0e762eb1f5684c700b7cbf1efa Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Thu, 18 Nov 2021 06:27:57 -0600 Subject: [PATCH 24/39] Fixed command line option processing for -j and -z --- rawspec.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/rawspec.c b/rawspec.c index f427350..05ba5d4 100644 --- a/rawspec.c +++ b/rawspec.c @@ -67,7 +67,7 @@ static struct option long_opts[] = { {"gpu", 1, NULL, 'g'}, {"help", 0, NULL, 'h'}, {"hdrs", 0, NULL, 'H'}, - {"ICS", 1, NULL, 'I'}, + {"ics", 1, NULL, 'i'}, {"fbh5", 0, NULL, 'j'}, {"nchan", 1, NULL, 'n'}, {"outidx", 1, NULL, 'o'}, @@ -153,9 +153,7 @@ int main(int argc, char *argv[]) int si; // Indexes the stems int fi; // Indexes the files for a given stem int bi; // Counts the blocks processed for a given file - int i; -int j, k; -char tmp[16]; + int i, j, k; void * pv; int fdin; int fdhdrs = -1; @@ -209,13 +207,16 @@ char tmp[16]; uint64_t total_bytes = 0; uint64_t total_ns = 0; + // Show librawspec version on startup + printf("rawspec using librawspec %s\n", rawspec_version_string()); + // Init rawspec context memset(&ctx, 0, sizeof(ctx)); ctx.Npolout[0] = 1; // others will be set later // Parse command line. argv0 = argv[0]; - while((opt=getopt_long(argc,argv,"a:b:d:f:g:Hjz:i:n:o:p:r:Ss:t:hv",long_opts,NULL))!=-1) { + while((opt=getopt_long(argc, argv, "a:b:d:f:g:HSjzs:i:n:o:p:r:t:hv", long_opts, NULL)) != -1) { switch (opt) { case 'h': // Help usage(argv0); @@ -276,8 +277,9 @@ char tmp[16]; save_headers = 1; break; - case 'i': // Incoherently sum exclusively - only_output_ics = 1; + case 'i': // Incoherent sum + printf("writing output for incoherent sum over all antennas\n"); + only_output_ics = 1; // will get reset if also splitting antennas ctx.incoherently_sum = 1; ctx.Naws = 1; // Count number of @@ -367,12 +369,18 @@ char tmp[16]; // If no stems given, print usage and exit if(argc == 0) { + fprintf(stderr, "error: a file stem must be specified\n"); usage(argv0); return 1; } - // Show librawspec version on startup - printf("rawspec using librawspec %s\n", rawspec_version_string()); + // If writing output files, show the format used + if(output_mode == RAWSPEC_FILE) { + if(flag_fbh5_output) + printf("writing output files in FBH5 format\n"); + else + printf("writing output files in SIGPROC Filterbank format\n"); + } // If schan is non-zero, nchan must be too if(schan != 0 && nchan == 0) { @@ -645,7 +653,6 @@ char tmp[16]; printf("Ignoring --splitant flag in network mode\n"); } if(only_output_ics){ - printf("Cancelling exclusivity of ICS output due to --splitant.\n"); only_output_ics = 0; } } @@ -1079,5 +1086,3 @@ char tmp[16]; return 0; } - - From 0ae1864cec66ef9919fbb937c62fada80d9c6521 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Thu, 18 Nov 2021 11:27:14 -0600 Subject: [PATCH 25/39] if(cb_data->fd && cb_data->h_pwrbuf && (cb_data->Nant > 1)) --- rawspec_file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rawspec_file.c b/rawspec_file.c index d9da1bd..37dc24c 100644 --- a/rawspec_file.c +++ b/rawspec_file.c @@ -114,7 +114,7 @@ void * dump_file_thread_func(void *arg) } // Multiple antennas, split output - if(cb_data->fd && cb_data->h_pwrbuf) { + if(cb_data->fd && cb_data->h_pwrbuf && (cb_data->Nant > 1)) { if(cb_data->per_ant_out) { size_t spectra_stride = cb_data->h_pwrbuf_size / (cb_data->Nds * sizeof(float)); size_t pol_stride = spectra_stride / cb_data->fb_hdr.nifs; From 3c89f644bc62036607c7284efeaa5a9d999bc837 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Thu, 18 Nov 2021 11:48:33 -0600 Subject: [PATCH 26/39] Avoid stderr for non-error message about Nbps value common.py/run_cmd() in rawspec_testing queries the contents of stderr as well as examining the exit_status. So, better to send the warning to stdout since it is not an error anyways. --- rawspec_gpu.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rawspec_gpu.cu b/rawspec_gpu.cu index 3be1d6f..f2b0eb0 100644 --- a/rawspec_gpu.cu +++ b/rawspec_gpu.cu @@ -484,10 +484,10 @@ int rawspec_initialize(rawspec_context * ctx) if(ctx->Nbps == 0) { ctx->Nbps = 8; } else if(ctx->Nbps != 8 && ctx->Nbps != 16) { - fprintf(stderr, - "number of bits per sample must be 8 or 16 (not %d), using 8 bps\n", + fprintf(stdout, + "WARNING: number of bits per sample = %d. Using 8 bps to appease the kernel\n", ctx->Nbps); - fflush(stderr); + fflush(stdout); NbpsIsExpanded = ctx->Nbps == 4; ctx->Nbps = 8; } From d2db1534425e51ea0049546044df3d400966cf12 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Thu, 18 Nov 2021 12:43:37 -0600 Subject: [PATCH 27/39] Modify a comment to emphasize "fine" vs "coarse" // number of fine filterbank channels (not coarse channels) int nchans; --- rawspec_fbutils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rawspec_fbutils.h b/rawspec_fbutils.h index 5f9150e..b1f0381 100644 --- a/rawspec_fbutils.h +++ b/rawspec_fbutils.h @@ -90,7 +90,7 @@ typedef struct { double fch1; // filterbank channel bandwidth (MHz) double foff; - // number of filterbank channels + // number of fine filterbank channels (not coarse channels) int nchans; // total number of beams int nbeams; From f8b96560f5cdf120a349a0d4df7ceb0315bab69f Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Thu, 18 Nov 2021 13:16:25 -0600 Subject: [PATCH 28/39] The WARNING message is redundant. "WARNING: number of bits per sample = 4. Using 8 bps to appease the kernel" is unnecessary since this also appears: --- rawspec_gpu.cu | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rawspec_gpu.cu b/rawspec_gpu.cu index f2b0eb0..c139237 100644 --- a/rawspec_gpu.cu +++ b/rawspec_gpu.cu @@ -484,10 +484,6 @@ int rawspec_initialize(rawspec_context * ctx) if(ctx->Nbps == 0) { ctx->Nbps = 8; } else if(ctx->Nbps != 8 && ctx->Nbps != 16) { - fprintf(stdout, - "WARNING: number of bits per sample = %d. Using 8 bps to appease the kernel\n", - ctx->Nbps); - fflush(stdout); NbpsIsExpanded = ctx->Nbps == 4; ctx->Nbps = 8; } From fbeb6464cd35bd389c47dfc9b6c9d33d51da3466 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Fri, 19 Nov 2021 06:50:26 -0600 Subject: [PATCH 29/39] Complete the removal of DEBUG_CALLBACKS as a compile-time switch from rawspec.c I left it in rawspec_socket.c and Makefile. I would rather first discuss this with David McMahon. From 13325cf45a70ce1dec94656a6ce64279e42d02e2 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Tue, 23 Nov 2021 11:38:12 -0600 Subject: [PATCH 30/39] Trial fix for issue #39 --- rawspec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/rawspec.c b/rawspec.c index 05ba5d4..9d3f032 100644 --- a/rawspec.c +++ b/rawspec.c @@ -1051,6 +1051,7 @@ int main(int argc, char *argv[]) } // each stem // Final cleanup + rawspec_reset_integration(&ctx); // Needed for issue #39 ? rawspec_cleanup(&ctx); if(ics_output_stem){ free(ics_output_stem); From 096daacf659c832cbc7d4ea720eb505406885e73 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Tue, 7 Dec 2021 14:06:03 -0800 Subject: [PATCH 31/39] Inserted timing into fbh5_write suring debug_callback --- fbh5_write.c | 8 ++++++++ rawspec_file.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fbh5_write.c b/fbh5_write.c index 17b6f00..507edfe 100644 --- a/fbh5_write.c +++ b/fbh5_write.c @@ -23,6 +23,8 @@ void fbh5_write(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, void * p_buffe hid_t filespace_id; // Identifier for a copy of the dataspace hsize_t selection[3]; // Current selection void *bufptr; // Pointer into the current dump + clock_t clock_1; // Debug time measurement + double cpu_time_used; // Debug time measurement /* * Initialise write loop. @@ -54,6 +56,7 @@ void fbh5_write(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, void * p_buffe selection[2] = p_fb_hdr->nchans; if(debug_callback) { + clock_1 = clock(); printf("fbh5_write: dump %ld, dump-tint %d, offset=(%lld, %lld, %lld), selection=(%lld, %lld, %lld), filesize=(%lld, %lld, %lld)\n", p_fbh5_ctx->dump_count, ii + 1, @@ -119,6 +122,11 @@ void fbh5_write(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, void * p_buffe */ bufptr += p_fbh5_ctx->tint_size; + if(debug_callback) { + cpu_time_used = ((double) (clock() - clock_1)) / CLOCKS_PER_SEC; + printf("fbh5_write: dump %ld E.T. = %.3f s\n", p_fbh5_ctx->dump_count, cpu_time_used); + } + /* * END write-loop for the current dump. */ diff --git a/rawspec_file.c b/rawspec_file.c index 37dc24c..3455d0e 100644 --- a/rawspec_file.c +++ b/rawspec_file.c @@ -146,7 +146,7 @@ void * dump_file_thread_func(void *arg) } // if(cb_data->per_ant_out) } // if(cb_data->fd && cb_data->h_pwrbuf) - // Multiple antennas, ICE output + // Multiple antennas, ICS output if(cb_data->fd_ics && cb_data->h_icsbuf) { if(cb_data->debug_callback) printf("dump_file_thread_func: write for ICS\n"); From fdccfdb0af2e6c5c979736a34504c2f6d0df1243 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Thu, 9 Dec 2021 16:24:37 -0600 Subject: [PATCH 32/39] Strategy change in open and write. 1. fbh5_open.c: Make use of 2 filters: bitshuffle and lz4. 2. fbh5_write.c: Write the entire sump in one HDF5 API call instead of integration by integration. Makefile editing will soon follow. --- fbh5_open.c | 110 ++++++++++++++++------------ fbh5_write.c | 198 +++++++++++++++++++++++++-------------------------- 2 files changed, 165 insertions(+), 143 deletions(-) diff --git a/fbh5_open.c b/fbh5_open.c index 1261770..ace1d41 100644 --- a/fbh5_open.c +++ b/fbh5_open.c @@ -8,10 +8,6 @@ #include "fbh5_defs.h" -unsigned COMPRESSION = 1; -unsigned SHUFFLE = 1; - - /*** Main entry point. ***/ @@ -23,16 +19,49 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p char wstr[256]; // sprintf target unsigned hdf5_majnum, hdf5_minnum, hdf5_relnum; // Version/release info for the HDF5 library - int deflate_level = 1; // 0=no deflation, 9=highest, 6=recommended - + // Filter data: + H5Z_filter_t filter_id_bitshuffle = 32008; + H5Z_filter_t filter_id_lz4 = 32004; + unsigned filter_flags = 0; + + + // Announce versions: H5get_libversion(&hdf5_majnum, &hdf5_minnum, &hdf5_relnum); - if(debug_callback) { - printf("fbh5_open: FBH5 path: %s\n", output_path); - printf("fbh5_open: HDF5 library version: %d.%d.%d\n", hdf5_majnum, hdf5_minnum, hdf5_relnum); - printf("fbh5_open: Creating dataspace dimensions using nifs=%d and nchans=%d\n", - p_fb_hdr->nifs, - p_fb_hdr->nchans); - } + printf("fbh5_open: FBH5 path: %s\n", output_path); + printf("fbh5_open: HDF5 library version: %d.%d.%d\n", hdf5_majnum, hdf5_minnum, hdf5_relnum); + printf("fbh5_open: Creating dataspace dimensions using nifs=%d and nchans=%d\n", + p_fb_hdr->nifs, + p_fb_hdr->nchans); + + /* + * Make sure that the Bitshuffle filter is available. + */ + if (H5Zfilter_avail(filter_id_bitshuffle) <= 0) + printf("*** fbhf_open: Filter bitshuffle is NOT available !!\n"); + else { + printf("fbhf_open: Filter bitshuffle is available.\n"); + if(H5Zget_filter_info(filter_id_bitshuffle, &filter_flags) < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: cannot fetch bitshuffle flags"); + if(H5Z_FILTER_CONFIG_ENCODE_ENABLED & filter_flags) + printf("fbhf_open: Filter bitshuffle encoder enabled.\n"); + else + printf("*** fbhf_open: Filter bitshuffle encoder disabled !!\n"); + } + + /* + * Make sure that the LZ4 filter is available. + */ + if (H5Zfilter_avail(filter_id_lz4) <= 0) + printf("*** fbhf_open: Filter LZF is NOT available !!\n"); + else { + printf("fbhf_open: Filter LZF is available.\n"); + if(H5Zget_filter_info(filter_id_lz4, &filter_flags) < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: cannot fetch LZ4 flags"); + if(H5Z_FILTER_CONFIG_ENCODE_ENABLED & filter_flags) + printf("fbhf_open: Filter LZ4 encoder enabled.\n"); + else + printf("*** fbhf_open: Filter LZ4 encoder disabled !!\n"); + } /* * Validate fb_hdr: nifs, nbits, nfpc, nchans. @@ -68,9 +97,9 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p * Open HDF5 file. Overwrite it if preexisting. */ p_fbh5_ctx->file_id = H5Fcreate(output_path, // Full path of output file - H5F_ACC_TRUNC, // Overwrite if preexisting. - H5P_DEFAULT, // Default creation property list - H5P_DEFAULT); // Default access property list + H5F_ACC_TRUNC, // Overwrite if preexisting. + H5P_DEFAULT, // Default creation property list + H5P_DEFAULT); // Default access property list if(p_fbh5_ctx->file_id < 0) { sprintf(wstr, "fbh5_open: H5Fcreate of '%s' FAILED", output_path); fbh5_oops(__FILE__, __LINE__, wstr); @@ -100,8 +129,8 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p * Create a dataspace which is extensible in the time dimension. */ p_fbh5_ctx->dataspace_id = H5Screate_simple(NDIMS, // Rank - p_fbh5_ctx->filesz_dims, // initial dimensions - max_dims); // maximum dimensions + p_fbh5_ctx->filesz_dims, // initial dimensions + max_dims); // maximum dimensions if(p_fbh5_ctx->dataspace_id < 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Screate_simple FAILED"); @@ -112,25 +141,6 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p if(dcpl < 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pcreate/dcpl FAILED"); - /* - * Add deflating to the dataset creation property list. - */ - if(COMPRESSION) { - if(SHUFFLE) { - status = H5Pset_shuffle(dcpl); - if(status < 0) - fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_shuffle FAILED"); - if(debug_callback) - printf("fbh5_open: Using shuffle deflation.\n"); - } - else - if(debug_callback) - printf("fbh5_open: Using gzip deflation.\n"); - status = H5Pset_deflate(dcpl, deflate_level); - if(status < 0) - fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_deflate FAILED"); - } - /* * Add chunking to the dataset creation property list. */ @@ -143,6 +153,18 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p if(debug_callback) printf("fbh5_open: Chunking (%lld, %lld, %lld) configured\n", cdims[0], cdims[1], cdims[2]); + /* + * Add the Bitshuffle filter and the LZF filter to the dataset creation property list. + */ + status = H5Pset_filter(dcpl, filter_id_bitshuffle, H5Z_FLAG_OPTIONAL, 0, NULL); // Bitshuffle Filter + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_filter FAILED"); + status = H5Pset_filter(dcpl, filter_id_lz4, H5Z_FLAG_OPTIONAL, 0, NULL); // LZF Filter + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_filter FAILED"); + if(debug_callback) + printf("fbh5_open: Using shuffle + LZF filter deflation.\n"); + /* * Define datatype for the data in the file. * We will store little endian values. @@ -164,13 +186,13 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p /* * Create the dataset. */ - p_fbh5_ctx->dataset_id = H5Dcreate(p_fbh5_ctx->file_id, // File handle - DATASETNAME, // Dataset name - p_fbh5_ctx->elem_type, // HDF5 data type - p_fbh5_ctx->dataspace_id, // Dataspace handle - H5P_DEFAULT, // - dcpl, // Dataset creation property list - H5P_DEFAULT); // Default access properties + p_fbh5_ctx->dataset_id = H5Dcreate(p_fbh5_ctx->file_id, // File handle + DATASETNAME, // Dataset name + p_fbh5_ctx->elem_type, // HDF5 data type + p_fbh5_ctx->dataspace_id, // Dataspace handle + H5P_DEFAULT, // + dcpl, // Dataset creation property list + H5P_DEFAULT); // Default access properties if(p_fbh5_ctx->dataset_id < 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Dcreate FAILED"); diff --git a/fbh5_write.c b/fbh5_write.c index 507edfe..a65f475 100644 --- a/fbh5_write.c +++ b/fbh5_write.c @@ -18,13 +18,12 @@ Main entry point. ***/ void fbh5_write(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, void * p_buffer, size_t bufsize, int debug_callback) { - herr_t status; // Status from HDF5 function call - size_t ntints; // Number of time integrations in the current dump - hid_t filespace_id; // Identifier for a copy of the dataspace - hsize_t selection[3]; // Current selection - void *bufptr; // Pointer into the current dump - clock_t clock_1; // Debug time measurement - double cpu_time_used; // Debug time measurement + herr_t status; // Status from HDF5 function call + size_t ntints; // Number of time integrations in the current dump + hid_t filespace_id; // Identifier for a copy of the dataspace + hsize_t selection[3]; // Current selection + clock_t clock_1; // Debug time measurement + double cpu_time_used; // Debug time measurement /* * Initialise write loop. @@ -33,105 +32,106 @@ void fbh5_write(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, void * p_buffe fbh5_show_context("fbh5_write", p_fbh5_ctx); ntints = bufsize / p_fbh5_ctx->tint_size; // Compute the number of time integrations in the current dump. p_fbh5_ctx->dump_count += 1; // Bump the dump count. - bufptr = p_buffer; // Point to the first time integration in the dump. /* - * BEGIN write-loop for the current dump. - * Write one time integration at a time. + * Bump the count of time integrations. + * One was already accounted for at open time - required by HDF5 library. + * So, + * If this is the very first write, bump by (ntints - 1); + * Else, bumpy by ntints. */ - for(int ii = 0; ii < ntints; ++ii) { - /* - * Bump the count of time integrations. - * If this is the very first write, no need to bump the file size - * because fbh5_open already accounted for the first time integration. - */ - if(p_fbh5_ctx->offset_dims[0] > 0) - p_fbh5_ctx->filesz_dims[0] += 1; - - /* - * Define the current slab selection in terms of its shape. - */ - selection[0] = 1; - selection[1] = p_fb_hdr->nifs; - selection[2] = p_fb_hdr->nchans; - - if(debug_callback) { - clock_1 = clock(); - printf("fbh5_write: dump %ld, dump-tint %d, offset=(%lld, %lld, %lld), selection=(%lld, %lld, %lld), filesize=(%lld, %lld, %lld)\n", - p_fbh5_ctx->dump_count, - ii + 1, - p_fbh5_ctx->offset_dims[0], - p_fbh5_ctx->offset_dims[1], - p_fbh5_ctx->offset_dims[2], - selection[0], - selection[1], - selection[2], - p_fbh5_ctx->filesz_dims[0], - p_fbh5_ctx->filesz_dims[1], - p_fbh5_ctx->filesz_dims[2]); - } - status = H5Dset_extent(p_fbh5_ctx->dataset_id, // Dataset handle - p_fbh5_ctx->filesz_dims); // New file size shape - if(status < 0) - fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Dset_extent/dataset_id FAILED"); - - /* - * Get filespace. - */ - filespace_id = H5Dget_space(p_fbh5_ctx->dataset_id); // Dataset handle - if(filespace_id < 0) - fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Dget_space FAILED"); - - /* - * Select the filespace hyperslab. - */ - status = H5Sselect_hyperslab(filespace_id, // Filespace handle - H5S_SELECT_SET, // Replace preexisting selection - p_fbh5_ctx->offset_dims, // Starting offset dimensions of first element - NULL, // Not "striding" - selection, // Selection dimensions - NULL); // Block parameter : default value - if(status < 0) - fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Sselect_hyperslab/filespace FAILED"); - - /* - * Write out current time integration to the hyperslab. - */ - status = H5Dwrite(p_fbh5_ctx->dataset_id, // Dataset handle - p_fbh5_ctx->elem_type, // HDF5 element type - p_fbh5_ctx->dataspace_id, // Dataspace handle - filespace_id, // Filespace_id - H5P_DEFAULT, // Default data transfer properties - bufptr); // Buffer holding the data - if(status < 0) - fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Dwrite FAILED"); - - /* - * Close temporary filespace handle. - */ - status = H5Sclose(filespace_id); - if(status < 0) - fbh5_oops(__FILE__, __LINE__, "fbh5_close H5Sclose/filespace_id FAILED\n"); - /* - * Point ahead for the next call to fbh5_write. - */ - p_fbh5_ctx->offset_dims[0] += 1; - - /* - * Bump the dump buffer pointer to the next time integration. - */ - bufptr += p_fbh5_ctx->tint_size; - - if(debug_callback) { - cpu_time_used = ((double) (clock() - clock_1)) / CLOCKS_PER_SEC; - printf("fbh5_write: dump %ld E.T. = %.3f s\n", p_fbh5_ctx->dump_count, cpu_time_used); - } + if(p_fbh5_ctx->offset_dims[0] > 0) + p_fbh5_ctx->filesz_dims[0] += ntints; + else + p_fbh5_ctx->filesz_dims[0] += (ntints - 1); /* - * END write-loop for the current dump. + * Define the current slab selection in terms of its shape. */ + selection[0] = ntints; + selection[1] = p_fb_hdr->nifs; + selection[2] = p_fb_hdr->nchans; + + if(debug_callback) { + printf("fbh5_write: dump %ld, offset=(%lld, %lld, %lld), selection=(%lld, %lld, %lld), filesize=(%lld, %lld, %lld)\n", + p_fbh5_ctx->dump_count, + p_fbh5_ctx->offset_dims[0], + p_fbh5_ctx->offset_dims[1], + p_fbh5_ctx->offset_dims[2], + selection[0], + selection[1], + selection[2], + p_fbh5_ctx->filesz_dims[0], + p_fbh5_ctx->filesz_dims[1], + p_fbh5_ctx->filesz_dims[2]); + clock_1 = clock(); + } + + /* + * Extend dataset. + */ + status = H5Dset_extent(p_fbh5_ctx->dataset_id, // Dataset handle + p_fbh5_ctx->filesz_dims); // New dataset shape + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Dset_extent/dataset_id FAILED"); + + /* + * Reset dataspace extent to match current slab selection. + */ + status = H5Sset_extent_simple(p_fbh5_ctx->dataspace_id, // Dataspace handle + NDIMS, // Repeat rank from previous API calls + selection, // New dataspace size shape + p_fbh5_ctx->filesz_dims); // Max dataspace dimensions + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Dset_extent/dataset_id FAILED"); + + /* + * Get filespace. + */ + filespace_id = H5Dget_space(p_fbh5_ctx->dataset_id); // Dataset handle + if(filespace_id < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Dget_space FAILED"); + + /* + * Select the filespace hyperslab. + */ + status = H5Sselect_hyperslab(filespace_id, // Filespace handle + H5S_SELECT_SET, // Replace preexisting selection + p_fbh5_ctx->offset_dims, // Starting offset dimensions of first element + NULL, // Not "striding" + selection, // Selection dimensions + NULL); // Block parameter : default value + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Sselect_hyperslab/filespace FAILED"); + + /* + * Write out current time integration to the hyperslab. + */ + status = H5Dwrite(p_fbh5_ctx->dataset_id, // Dataset handle + p_fbh5_ctx->elem_type, // HDF5 element type + p_fbh5_ctx->dataspace_id, // Dataspace handle + filespace_id, // Filespace_id + H5P_DEFAULT, // Default data transfer properties + p_buffer); // Buffer holding the data + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_write: H5Dwrite FAILED"); + + /* + * Point ahead for the next call to fbh5_write. + */ + p_fbh5_ctx->offset_dims[0] += ntints; + + /* + * Close temporary filespace handle. + */ + status = H5Sclose(filespace_id); + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_close H5Sclose/filespace_id FAILED\n"); + if(debug_callback) { + cpu_time_used = ((double) (clock() - clock_1)) / CLOCKS_PER_SEC; + printf("fbh5_write: dump %ld E.T. = %.3f s\n", p_fbh5_ctx->dump_count, cpu_time_used); } - + /* * Bump counters. */ From 31d65092aeff1c797347aec2f7c7f578410b70e1 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Fri, 10 Dec 2021 06:52:01 -0800 Subject: [PATCH 33/39] Disallow -i and -S together --- Makefile | 6 +++++- fbh5_open.c | 22 ++++------------------ rawspec.c | 7 +++++++ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 0856a6d..e6c1177 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,11 @@ INCDIR_H5= /usr/include/hdf5/serial/ LIBDIR_h5= /usr/lib/x86_64-linux-gnu/hdf5/serial/ LIBHDF5= :libhdf5.so LIBHDF5_HL= :libhdf5_hl.so -LINKH5:= -L$(LIBDIR) -l $(LIBHDF5) -l $(LIBHDF5_HL) +PLUGIN_DIR= /usr/local/hdf5/lib/plugin +PLUGIN_DIR= $(HOME)/.local/lib/python3.7/site-packages/hdf5plugin/plugins +LIBBSHUF= :libh5bshuf.so +LIBLZF= :libh5lz4.so +LINKH5:= -L$(LIBDIR) -l $(LIBHDF5) -l $(LIBHDF5_HL) -L $(PLUGIN_DIR) -l $(LIBBSHUF) -l $(LIBLZF) # End HDF5 definitions CUDA_DIR ?= $(CUDA_ROOT) diff --git a/fbh5_open.c b/fbh5_open.c index ace1d41..6613cf9 100644 --- a/fbh5_open.c +++ b/fbh5_open.c @@ -38,30 +38,16 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p */ if (H5Zfilter_avail(filter_id_bitshuffle) <= 0) printf("*** fbhf_open: Filter bitshuffle is NOT available !!\n"); - else { + else printf("fbhf_open: Filter bitshuffle is available.\n"); - if(H5Zget_filter_info(filter_id_bitshuffle, &filter_flags) < 0) - fbh5_oops(__FILE__, __LINE__, "fbh5_open: cannot fetch bitshuffle flags"); - if(H5Z_FILTER_CONFIG_ENCODE_ENABLED & filter_flags) - printf("fbhf_open: Filter bitshuffle encoder enabled.\n"); - else - printf("*** fbhf_open: Filter bitshuffle encoder disabled !!\n"); - } /* * Make sure that the LZ4 filter is available. */ if (H5Zfilter_avail(filter_id_lz4) <= 0) printf("*** fbhf_open: Filter LZF is NOT available !!\n"); - else { + else printf("fbhf_open: Filter LZF is available.\n"); - if(H5Zget_filter_info(filter_id_lz4, &filter_flags) < 0) - fbh5_oops(__FILE__, __LINE__, "fbh5_open: cannot fetch LZ4 flags"); - if(H5Z_FILTER_CONFIG_ENCODE_ENABLED & filter_flags) - printf("fbhf_open: Filter LZ4 encoder enabled.\n"); - else - printf("*** fbhf_open: Filter LZ4 encoder disabled !!\n"); - } /* * Validate fb_hdr: nifs, nbits, nfpc, nchans. @@ -156,10 +142,10 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p /* * Add the Bitshuffle filter and the LZF filter to the dataset creation property list. */ - status = H5Pset_filter(dcpl, filter_id_bitshuffle, H5Z_FLAG_OPTIONAL, 0, NULL); // Bitshuffle Filter + status = H5Pset_filter(dcpl, filter_id_bitshuffle, H5Z_FLAG_MANDATORY, 0, NULL); // Bitshuffle Filter if(status < 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_filter FAILED"); - status = H5Pset_filter(dcpl, filter_id_lz4, H5Z_FLAG_OPTIONAL, 0, NULL); // LZF Filter + status = H5Pset_filter(dcpl, filter_id_lz4, H5Z_FLAG_MANDATORY, 0, NULL); // LZF Filter if(status < 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_filter FAILED"); if(debug_callback) diff --git a/rawspec.c b/rawspec.c index 9d3f032..a80b05b 100644 --- a/rawspec.c +++ b/rawspec.c @@ -374,6 +374,13 @@ int main(int argc, char *argv[]) return 1; } + // Currently, there are potential conflicts in running -i and -S concurrently. + if(ctx.incoherently_sum == 1 && per_ant_out == 1) { + fprintf(stderr, "PLEASE NOTE: Currently, there are potential conflicts in running -i and -S concurrently.\n"); + fprintf(stderr, "PLEASE NOTE: -S (split antennas) is being ignored.\n"); + per_ant_out = 0; + } + // If writing output files, show the format used if(output_mode == RAWSPEC_FILE) { if(flag_fbh5_output) From d2c6e076ed792aa8dfee181ed39c03231e823b6f Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Fri, 10 Dec 2021 07:58:00 -0800 Subject: [PATCH 34/39] Finished investigation of blimpy compression options and applied results to fbh5_open.c --- fbh5_open.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/fbh5_open.c b/fbh5_open.c index 6613cf9..42babb5 100644 --- a/fbh5_open.c +++ b/fbh5_open.c @@ -19,11 +19,15 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p char wstr[256]; // sprintf target unsigned hdf5_majnum, hdf5_minnum, hdf5_relnum; // Version/release info for the HDF5 library - // Filter data: + // Filter identities: H5Z_filter_t filter_id_bitshuffle = 32008; H5Z_filter_t filter_id_lz4 = 32004; - unsigned filter_flags = 0; + // Bitshuffle options: + unsigned bitshuffle_opts[] = {0, 2}; + // Ref: def __init__ in class Bitshuffle in https://github.com/silx-kit/hdf5plugin/blob/main/src/hdf5plugin/__init__.py + // 0 = take default like blimpy + // 2 = use lz4 like blimpy // Announce versions: H5get_libversion(&hdf5_majnum, &hdf5_minnum, &hdf5_relnum); @@ -45,9 +49,9 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p * Make sure that the LZ4 filter is available. */ if (H5Zfilter_avail(filter_id_lz4) <= 0) - printf("*** fbhf_open: Filter LZF is NOT available !!\n"); + printf("*** fbhf_open: Filter LZ4 is NOT available !!\n"); else - printf("fbhf_open: Filter LZF is available.\n"); + printf("fbhf_open: Filter LZ4 is available.\n"); /* * Validate fb_hdr: nifs, nbits, nfpc, nchans. @@ -136,20 +140,17 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p status = H5Pset_chunk(dcpl, NDIMS, cdims); if(status != 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_chunk FAILED"); - if(debug_callback) - printf("fbh5_open: Chunking (%lld, %lld, %lld) configured\n", cdims[0], cdims[1], cdims[2]); + printf("fbh5_open: Chunk dimensions = (%lld, %lld, %lld)\n", cdims[0], cdims[1], cdims[2]); /* - * Add the Bitshuffle filter and the LZF filter to the dataset creation property list. + * Add the Bitshuffle and LZ4 filters to the dataset creation property list. */ - status = H5Pset_filter(dcpl, filter_id_bitshuffle, H5Z_FLAG_MANDATORY, 0, NULL); // Bitshuffle Filter + status = H5Pset_filter(dcpl, filter_id_bitshuffle, H5Z_FLAG_MANDATORY, 0, bitshuffle_opts); // Bitshuffle Filter if(status < 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_filter FAILED"); - status = H5Pset_filter(dcpl, filter_id_lz4, H5Z_FLAG_MANDATORY, 0, NULL); // LZF Filter + status = H5Pset_filter(dcpl, filter_id_lz4, H5Z_FLAG_MANDATORY, 0, NULL); // LZ4 Filter if(status < 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_filter FAILED"); - if(debug_callback) - printf("fbh5_open: Using shuffle + LZF filter deflation.\n"); /* * Define datatype for the data in the file. From 09fc20e6680a84ce6f8b634c047e26e3b48b133c Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Fri, 10 Dec 2021 10:44:58 -0800 Subject: [PATCH 35/39] Remove references to plugins. --- Makefile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Makefile b/Makefile index e6c1177..0856a6d 100644 --- a/Makefile +++ b/Makefile @@ -10,11 +10,7 @@ INCDIR_H5= /usr/include/hdf5/serial/ LIBDIR_h5= /usr/lib/x86_64-linux-gnu/hdf5/serial/ LIBHDF5= :libhdf5.so LIBHDF5_HL= :libhdf5_hl.so -PLUGIN_DIR= /usr/local/hdf5/lib/plugin -PLUGIN_DIR= $(HOME)/.local/lib/python3.7/site-packages/hdf5plugin/plugins -LIBBSHUF= :libh5bshuf.so -LIBLZF= :libh5lz4.so -LINKH5:= -L$(LIBDIR) -l $(LIBHDF5) -l $(LIBHDF5_HL) -L $(PLUGIN_DIR) -l $(LIBBSHUF) -l $(LIBLZF) +LINKH5:= -L$(LIBDIR) -l $(LIBHDF5) -l $(LIBHDF5_HL) # End HDF5 definitions CUDA_DIR ?= $(CUDA_ROOT) From e50157ed8b915f608836b48e930b08db82ca53af Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Sat, 11 Dec 2021 05:46:07 -0800 Subject: [PATCH 36/39] One filter (bitshuffle) is sufficient. --- fbh5_open.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/fbh5_open.c b/fbh5_open.c index 42babb5..36e82c3 100644 --- a/fbh5_open.c +++ b/fbh5_open.c @@ -21,7 +21,6 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p // Filter identities: H5Z_filter_t filter_id_bitshuffle = 32008; - H5Z_filter_t filter_id_lz4 = 32004; // Bitshuffle options: unsigned bitshuffle_opts[] = {0, 2}; @@ -45,14 +44,6 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p else printf("fbhf_open: Filter bitshuffle is available.\n"); - /* - * Make sure that the LZ4 filter is available. - */ - if (H5Zfilter_avail(filter_id_lz4) <= 0) - printf("*** fbhf_open: Filter LZ4 is NOT available !!\n"); - else - printf("fbhf_open: Filter LZ4 is available.\n"); - /* * Validate fb_hdr: nifs, nbits, nfpc, nchans. */ @@ -145,10 +136,7 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p /* * Add the Bitshuffle and LZ4 filters to the dataset creation property list. */ - status = H5Pset_filter(dcpl, filter_id_bitshuffle, H5Z_FLAG_MANDATORY, 0, bitshuffle_opts); // Bitshuffle Filter - if(status < 0) - fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_filter FAILED"); - status = H5Pset_filter(dcpl, filter_id_lz4, H5Z_FLAG_MANDATORY, 0, NULL); // LZ4 Filter + status = H5Pset_filter(dcpl, filter_id_bitshuffle, H5Z_FLAG_MANDATORY, 2, bitshuffle_opts); // Bitshuffle Filter if(status < 0) fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_filter FAILED"); @@ -206,4 +194,4 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p if(debug_callback) fbh5_show_context("fbh5_open", p_fbh5_ctx); -} +} \ No newline at end of file From 7372f017ca1ef51649c6c5fb4439d9b90dd8e4e0 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Sat, 11 Dec 2021 06:49:05 -0800 Subject: [PATCH 37/39] Created `locplug` to announce where the HDF5 plugin directory is located. --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0856a6d..aa1f404 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ DATADIR = $(PREFIX)/share # Begin HDF5 definitions INCDIR_H5= /usr/include/hdf5/serial/ -LIBDIR_h5= /usr/lib/x86_64-linux-gnu/hdf5/serial/ +#LIBDIR_h5= /usr/lib/x86_64-linux-gnu/hdf5/serial/ +LIBDIR_h5= /usr/local/lib LIBHDF5= :libhdf5.so LIBHDF5_HL= :libhdf5_hl.so LINKH5:= -L$(LIBDIR) -l $(LIBHDF5) -l $(LIBHDF5_HL) @@ -58,7 +59,7 @@ endif # Possibly (re-)build rawspec_version.h $(shell $(SHELL) gen_version.sh) -all: rawspec rawspectest fileiotest +all: rawspec rawspectest fileiotest locplug # Dependencoes are simple enough to manage manually (for now) fileiotest.o: rawspec.h @@ -102,9 +103,12 @@ fileiotest: fileiotest.o rawspec_fbutils: rawspec_fbutils.c rawspec_fbutils.h $(CC) -o $@ -DFBUTILS_TEST -ggdb -O0 $< -lm +locplug: locplug.c + $(CC) -o locplug locplug.c -lm + install: rawspec rawspec.h librawspec.so mkdir -p $(BINDIR) - cp -p rawspec $(BINDIR) + cp -p rawspec locplug $(BINDIR) mkdir -p $(INCDIR) cp -p rawspec.h $(INCDIR) cp -p rawspec_fbutils.h $(INCDIR) From 2383538e143500fc6883333e3bfdd222c83a776b Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Sat, 11 Dec 2021 14:06:31 -0800 Subject: [PATCH 38/39] Move locplug into rawspec.c --- Makefile | 7 ++----- fbh5_open.c | 26 ++++++++++++++++++++++++++ rawspec.c | 9 +++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index aa1f404..34de2ca 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ endif # Possibly (re-)build rawspec_version.h $(shell $(SHELL) gen_version.sh) -all: rawspec rawspectest fileiotest locplug +all: rawspec rawspectest fileiotest # Dependencoes are simple enough to manage manually (for now) fileiotest.o: rawspec.h @@ -103,12 +103,9 @@ fileiotest: fileiotest.o rawspec_fbutils: rawspec_fbutils.c rawspec_fbutils.h $(CC) -o $@ -DFBUTILS_TEST -ggdb -O0 $< -lm -locplug: locplug.c - $(CC) -o locplug locplug.c -lm - install: rawspec rawspec.h librawspec.so mkdir -p $(BINDIR) - cp -p rawspec locplug $(BINDIR) + cp -p rawspec $(BINDIR) mkdir -p $(INCDIR) cp -p rawspec.h $(INCDIR) cp -p rawspec_fbutils.h $(INCDIR) diff --git a/fbh5_open.c b/fbh5_open.c index 36e82c3..90f421f 100644 --- a/fbh5_open.c +++ b/fbh5_open.c @@ -8,6 +8,9 @@ #include "fbh5_defs.h" +int CACHE_SPECS = 0; + + /*** Main entry point. ***/ @@ -18,6 +21,11 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p herr_t status; // Status from HDF5 function call char wstr[256]; // sprintf target unsigned hdf5_majnum, hdf5_minnum, hdf5_relnum; // Version/release info for the HDF5 library + + // Caching + hid_t fapl; // File access property list identifier + size_t fcache_nslots = 521; // Hash table number of slots. Default value. + size_t fcache_nbytes = 0; // Cache size in bytes // Filter identities: H5Z_filter_t filter_id_bitshuffle = 32008; @@ -86,6 +94,24 @@ void fbh5_open(fbh5_context_t * p_fbh5_ctx, fb_hdr_t * p_fb_hdr, char * output_p fbh5_oops(__FILE__, __LINE__, wstr); } + /* + * If specifying file-level caching specifications, do so now. + */ + + if(CACHE_SPECS == 1) { + fcache_nbytes = p_fbh5_ctx->tint_size + 42; + fapl = H5Fget_access_plist(p_fbh5_ctx->file_id); + if(fapl < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Fget_access_plist FAILED"); + status = H5Pset_cache(fapl, + 0, // ignored + fcache_nslots, // Hash table slot count + fcache_nbytes, // File cache size in bytes + 1.0); // Rawspec writes data only once. Never reads. + if(status < 0) + fbh5_oops(__FILE__, __LINE__, "fbh5_open: H5Pset_cache FAILED"); + } + /* * Write the file-level metadata attributes. */ diff --git a/rawspec.c b/rawspec.c index a80b05b..e80f769 100644 --- a/rawspec.c +++ b/rawspec.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "rawspec.h" #include "rawspec_file.h" @@ -34,6 +35,12 @@ #define DEBUG_CALLBACKS (0) #endif +void locplug() { + fprintf(stderr,"\nThe plugin directory is %s.\n", H5_DEFAULT_PLUGINDIR); + fprintf(stderr,"Found the name in the H5pubconf.h include file.\n"); + fprintf(stderr,"Please copy the bitshuffle plugin to that directory if you haven't already done so.\n\n"); +} + // Reads `bytes_to_read` bytes from `fd` into the buffer pointed to by `buf`. // Returns the total bytes read or -1 on error. A non-negative return value // will be less than `bytes_to_read` only of EOF is reached. @@ -115,6 +122,7 @@ void usage(const char *argv0) { " -v, --version Show version and exit\n" , bname ); + locplug(); } int open_headers_file(const char * dest, const char *stem) @@ -351,6 +359,7 @@ int main(int argc, char *argv[]) case 'v': // Version printf("rawspec %s\n", STRINGIFY(RAWSPEC_VERSION)); printf("librawspec %s\n", rawspec_version_string()); + locplug(); return 0; break; From 67c1e563d2716a2d4cb8cd037b56a4693f0fd4bc Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Thu, 16 Dec 2021 09:10:14 -0600 Subject: [PATCH 39/39] Remove unnecessary reference to H5pubconf.h include file --- rawspec.c | 1 - 1 file changed, 1 deletion(-) diff --git a/rawspec.c b/rawspec.c index e80f769..744c072 100644 --- a/rawspec.c +++ b/rawspec.c @@ -37,7 +37,6 @@ void locplug() { fprintf(stderr,"\nThe plugin directory is %s.\n", H5_DEFAULT_PLUGINDIR); - fprintf(stderr,"Found the name in the H5pubconf.h include file.\n"); fprintf(stderr,"Please copy the bitshuffle plugin to that directory if you haven't already done so.\n\n"); }