From 9cf74d261262cfe701afd64e3c91641e091c97fc Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 6 Mar 2023 23:13:20 -0600 Subject: [PATCH 001/216] remove unnecessary install block from CMakeLists.txt --- src/server/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 7d59bafb9..449b64f10 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -54,11 +54,11 @@ add_executable(pdc_server.exe ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_hist_pkg.c ) -install( - TARGETS - pdc_server.exe - DESTINATION ${PDC_INSTALL_BIN_DIR} -) +#install( +# TARGETS +# pdc_server.exe +# DESTINATION ${PDC_INSTALL_BIN_DIR} +#) if(NOT ${PDC_INSTALL_BIN_DIR} MATCHES ${PROJECT_BINARY_DIR}/bin) install( From fe1f8b44995bc0dabd3b957e1032c2da26f56fdd Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 20 Mar 2023 11:10:40 -0500 Subject: [PATCH 002/216] update output --- src/tests/kvtag_add_get_scale.c | 42 ++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index b59125d46..e7e7657e1 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -58,6 +58,19 @@ assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) return 1; } + +uint64_t +atoui64(char *arg) { + char *endptr; + uint64_t num = strtoull(arg, &endptr, 10); + + if (*endptr != '\0') { + printf("Invalid input: %s\n", arg); + return 1; + } + return num; +} + void print_usage(char *name) { @@ -69,8 +82,9 @@ main(int argc, char *argv[]) { pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t * obj_ids; - int n_obj, n_add_tag, n_query, my_obj, my_obj_s, my_add_tag, my_query, my_add_tag_s, my_query_s; - int proc_num, my_rank, i, v; + uint64_t n_obj, n_add_tag, n_query, my_obj, my_obj_s, my_add_tag, my_query, my_add_tag_s, my_query_s; + int proc_num, my_rank + uint64_t i, v; char obj_name[128]; double stime, total_time; pdc_kvtag_t kvtag; @@ -86,9 +100,9 @@ main(int argc, char *argv[]) print_usage(argv[0]); goto done; } - n_obj = atoi(argv[1]); - n_add_tag = atoi(argv[2]); - n_query = atoi(argv[3]); + n_obj = atoui64(argv[1]); + n_add_tag = atoui64(argv[2]); + n_query = atoui64(argv[3]); if (n_add_tag > n_obj || n_query > n_obj) { if (my_rank == 0) @@ -101,7 +115,7 @@ main(int argc, char *argv[]) assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); if (my_rank == 0) - printf("Create %d obj, %d tags, query %d\n", my_obj, my_add_tag, my_query); + printf("Create %llu obj, %llu tags, query %llu\n", my_obj, my_add_tag, my_query); // create a pdc pdc = PDCinit("pdc"); @@ -124,14 +138,14 @@ main(int argc, char *argv[]) // Create a number of objects, add at least one tag to that object obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); for (i = 0; i < my_obj; i++) { - sprintf(obj_name, "obj%d", my_obj_s + i); + sprintf(obj_name, "obj%llu", my_obj_s + i); obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); if (obj_ids[i] <= 0) printf("Fail to create object @ line %d!\n", __LINE__); } if (my_rank == 0) - printf("Created %d objects\n", n_obj); + printf("Created %llu objects\n", n_obj); // Add tags kvtag.name = "Group"; @@ -145,7 +159,7 @@ main(int argc, char *argv[]) for (i = 0; i < my_add_tag; i++) { v = i + my_add_tag_s; if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) - printf("fail to add a kvtag to o%d\n", i + my_obj_s); + printf("fail to add a kvtag to o%llu\n", i + my_obj_s); } #ifdef ENABLE_MPI @@ -153,7 +167,7 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to add tags to %d objects: %.4f\n", n_add_tag, total_time); + printf("Total time to add tags to %llu objects: %.4f\n", n_add_tag, total_time); values = (pdc_kvtag_t **)calloc(my_query, sizeof(pdc_kvtag_t *)); @@ -163,7 +177,7 @@ main(int argc, char *argv[]) #endif for (i = 0; i < my_query; i++) { if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_size) < 0) - printf("fail to get a kvtag from o%d\n", i + my_query_s); + printf("fail to get a kvtag from o%llu\n", i + my_query_s); } #ifdef ENABLE_MPI @@ -171,13 +185,13 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to retrieve tags from %d objects: %.4f\n", n_query, total_time); + printf("Total time to retrieve tags from %llu objects: %.4f\n", n_query, total_time); fflush(stdout); for (i = 0; i < my_query; i++) { if (*(int *)(values[i]->value) != i + my_add_tag_s) - printf("Error with retrieved tag from o%d\n", i + my_query_s); + printf("Error with retrieved tag from o%llu\n", i + my_query_s); PDC_free_kvtag(&values[i]); } @@ -186,7 +200,7 @@ main(int argc, char *argv[]) // close first object for (i = 0; i < my_obj; i++) { if (PDCobj_close(obj_ids[i]) < 0) - printf("fail to close object o%d\n", i + my_obj_s); + printf("fail to close object o%llu\n", i + my_obj_s); } // close a container From 596a4b88904077ef29e123202c714cbc9fcc5a7f Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 20 Mar 2023 12:52:57 -0500 Subject: [PATCH 003/216] Revert "update output" This reverts commit fe1f8b44995bc0dabd3b957e1032c2da26f56fdd. --- src/tests/kvtag_add_get_scale.c | 42 +++++++++++---------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index e7e7657e1..b59125d46 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -58,19 +58,6 @@ assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) return 1; } - -uint64_t -atoui64(char *arg) { - char *endptr; - uint64_t num = strtoull(arg, &endptr, 10); - - if (*endptr != '\0') { - printf("Invalid input: %s\n", arg); - return 1; - } - return num; -} - void print_usage(char *name) { @@ -82,9 +69,8 @@ main(int argc, char *argv[]) { pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t * obj_ids; - uint64_t n_obj, n_add_tag, n_query, my_obj, my_obj_s, my_add_tag, my_query, my_add_tag_s, my_query_s; - int proc_num, my_rank - uint64_t i, v; + int n_obj, n_add_tag, n_query, my_obj, my_obj_s, my_add_tag, my_query, my_add_tag_s, my_query_s; + int proc_num, my_rank, i, v; char obj_name[128]; double stime, total_time; pdc_kvtag_t kvtag; @@ -100,9 +86,9 @@ main(int argc, char *argv[]) print_usage(argv[0]); goto done; } - n_obj = atoui64(argv[1]); - n_add_tag = atoui64(argv[2]); - n_query = atoui64(argv[3]); + n_obj = atoi(argv[1]); + n_add_tag = atoi(argv[2]); + n_query = atoi(argv[3]); if (n_add_tag > n_obj || n_query > n_obj) { if (my_rank == 0) @@ -115,7 +101,7 @@ main(int argc, char *argv[]) assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); if (my_rank == 0) - printf("Create %llu obj, %llu tags, query %llu\n", my_obj, my_add_tag, my_query); + printf("Create %d obj, %d tags, query %d\n", my_obj, my_add_tag, my_query); // create a pdc pdc = PDCinit("pdc"); @@ -138,14 +124,14 @@ main(int argc, char *argv[]) // Create a number of objects, add at least one tag to that object obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); for (i = 0; i < my_obj; i++) { - sprintf(obj_name, "obj%llu", my_obj_s + i); + sprintf(obj_name, "obj%d", my_obj_s + i); obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); if (obj_ids[i] <= 0) printf("Fail to create object @ line %d!\n", __LINE__); } if (my_rank == 0) - printf("Created %llu objects\n", n_obj); + printf("Created %d objects\n", n_obj); // Add tags kvtag.name = "Group"; @@ -159,7 +145,7 @@ main(int argc, char *argv[]) for (i = 0; i < my_add_tag; i++) { v = i + my_add_tag_s; if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) - printf("fail to add a kvtag to o%llu\n", i + my_obj_s); + printf("fail to add a kvtag to o%d\n", i + my_obj_s); } #ifdef ENABLE_MPI @@ -167,7 +153,7 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to add tags to %llu objects: %.4f\n", n_add_tag, total_time); + printf("Total time to add tags to %d objects: %.4f\n", n_add_tag, total_time); values = (pdc_kvtag_t **)calloc(my_query, sizeof(pdc_kvtag_t *)); @@ -177,7 +163,7 @@ main(int argc, char *argv[]) #endif for (i = 0; i < my_query; i++) { if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_size) < 0) - printf("fail to get a kvtag from o%llu\n", i + my_query_s); + printf("fail to get a kvtag from o%d\n", i + my_query_s); } #ifdef ENABLE_MPI @@ -185,13 +171,13 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to retrieve tags from %llu objects: %.4f\n", n_query, total_time); + printf("Total time to retrieve tags from %d objects: %.4f\n", n_query, total_time); fflush(stdout); for (i = 0; i < my_query; i++) { if (*(int *)(values[i]->value) != i + my_add_tag_s) - printf("Error with retrieved tag from o%llu\n", i + my_query_s); + printf("Error with retrieved tag from o%d\n", i + my_query_s); PDC_free_kvtag(&values[i]); } @@ -200,7 +186,7 @@ main(int argc, char *argv[]) // close first object for (i = 0; i < my_obj; i++) { if (PDCobj_close(obj_ids[i]) < 0) - printf("fail to close object o%llu\n", i + my_obj_s); + printf("fail to close object o%d\n", i + my_obj_s); } // close a container From 041d368d9aa3d7a0624ad5959941a99a2ed7bfa2 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 20 Mar 2023 13:07:10 -0500 Subject: [PATCH 004/216] build kvtag_add_get_scale --- src/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 965a39fed..aae44e8f2 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -63,7 +63,7 @@ set(PROGRAMS # data_server_meta_test # kvtag_add_get # kvtag_get -# kvtag_add_get_scale + kvtag_add_get_scale # kvtag_query # kvtag_query_scale # obj_transformation From b14da3521f0404958e6e34e6485f29e595d802b1 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 20 Mar 2023 13:40:39 -0500 Subject: [PATCH 005/216] comment off free --- src/tests/kvtag_add_get_scale.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index b59125d46..464ab45b2 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -175,12 +175,12 @@ main(int argc, char *argv[]) fflush(stdout); - for (i = 0; i < my_query; i++) { - if (*(int *)(values[i]->value) != i + my_add_tag_s) - printf("Error with retrieved tag from o%d\n", i + my_query_s); + // for (i = 0; i < my_query; i++) { + // if (*(int *)(values[i]->value) != i + my_add_tag_s) + // printf("Error with retrieved tag from o%d\n", i + my_query_s); - PDC_free_kvtag(&values[i]); - } + // PDC_free_kvtag(&values[i]); + // } free(values); // close first object From 31d15313ed566218957d83c357d309d882eb1a4d Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 20 Mar 2023 14:06:28 -0500 Subject: [PATCH 006/216] update code --- src/tests/kvtag_add_get_scale.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index 464ab45b2..7f7745a7c 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -175,12 +175,12 @@ main(int argc, char *argv[]) fflush(stdout); - // for (i = 0; i < my_query; i++) { - // if (*(int *)(values[i]->value) != i + my_add_tag_s) - // printf("Error with retrieved tag from o%d\n", i + my_query_s); + for (i = 0; i < my_query; i++) { + if (*(int *)(values[i]->value) != i + my_add_tag_s) + printf("Error with retrieved tag from o%d\n", i + my_query_s); - // PDC_free_kvtag(&values[i]); - // } + // PDC_free_kvtag(&values[i]); + } free(values); // close first object From d81440c870c1ca7dc773dcbddcc718852467ed21 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 22 Mar 2023 23:45:53 -0500 Subject: [PATCH 007/216] 1. kvtag_scale_add_get added \n 2. uint64_t support for obj/tag/query count \n 3. moving work assigning block downwards right before creating objects \n 4. everything is tested working --- src/tests/kvtag_scale_add_get.c | 230 ++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 src/tests/kvtag_scale_add_get.c diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c new file mode 100644 index 000000000..1493a7d6d --- /dev/null +++ b/src/tests/kvtag_scale_add_get.c @@ -0,0 +1,230 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "pdc_client_connect.h" + +uint64_t +atoui64(char *arg) { + char *endptr; + uint64_t num = strtoull(arg, &endptr, 10); + + if (*endptr != '\0') { + printf("Invalid input: %s\n", arg); + return 1; + } + return num; +} + +int +assign_work_to_rank(int rank, int size, uint64_t nwork, uint64_t *my_count, uint64_t *my_start) +{ + if (rank > size || my_count == NULL || my_start == NULL) { + printf("assign_work_to_rank(): Error with input!\n"); + return -1; + } + if (nwork < size) { + if (rank < nwork) + *my_count = 1; + else + *my_count = 0; + (*my_start) = rank * (*my_count); + } + else { + (*my_count) = nwork / size; + (*my_start) = rank * (*my_count); + + // Last few ranks may have extra work + if (rank >= size - nwork % size) { + (*my_count)++; + (*my_start) += (rank - (size - nwork % size)); + } + } + + return 1; +} + +void +print_usage(char *name) +{ + printf("%s n_obj n_add_tag n_query\n", name); +} + +int +main(int argc, char *argv[]) +{ + pdcid_t pdc, cont_prop, cont, obj_prop; + pdcid_t *obj_ids; + uint64_t n_obj, n_obj_incr, my_obj, my_obj_s; + uint64_t n_add_tag, my_add_tag, my_add_tag_s; + uint64_t n_query, my_query, my_query_s; + uint64_t i, v; + int proc_num, my_rank; + char obj_name[128]; + char tag_name[128]; + double stime, total_time; + pdc_kvtag_t kvtag; + void **values; + size_t value_size; +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &proc_num); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#endif + if (argc != 5) { + if (my_rank == 0) + print_usage(argv[0]); + goto done; + } + n_obj = atoui64(argv[1]); + n_obj_incr = atoui64(argv[2]); + n_add_tag = atoui64(argv[3]); + n_query = atoui64(argv[4]); + + if (n_add_tag > n_obj || n_query > n_obj) { + if (my_rank == 0) + printf("n_add_tag or n_query larger than n_obj! Exiting...\n"); + goto done; + } + + if (my_rank == 0) + printf("Create %d obj, %d tags, query %d\n", my_obj, my_add_tag, my_query); + + // create a pdc + pdc = PDCinit("pdc"); + + // create a container property + cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); + + // create a container + cont = PDCcont_create("c1", cont_prop); + if (cont <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); + + // create an object property + obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); + if (obj_prop <= 0) + printf("Fail to create object property @ line %d!\n", __LINE__); + + assign_work_to_rank(my_rank, proc_num, n_add_tag, &my_add_tag, &my_add_tag_s); + assign_work_to_rank(my_rank, proc_num, n_query, &my_query, &my_query_s); + assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); + + // Create a number of objects, add at least one tag to that object + obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + for (i = 0; i < my_obj; i++) { + sprintf(obj_name, "obj%d", my_obj_s + i); + obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); + if (obj_ids[i] <= 0) + printf("Fail to create object @ line %d!\n", __LINE__); + } + + if (my_rank == 0) + printf("Created %d objects\n", n_obj); + + // Add tags + kvtag.name = "Group"; + kvtag.value = (void *)&v; + kvtag.size = sizeof(uint64_t); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + for (i = 0; i < my_add_tag; i++) { + v = i + my_add_tag_s; + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) + printf("fail to add a kvtag to o%d\n", i + my_obj_s); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) + printf("Total time to add tags to %d objects: %.4f\n", n_add_tag, total_time); + + values = (void **)calloc(my_query, sizeof(void *)); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + for (i = 0; i < my_query; i++) { + if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_size) < 0) + printf("fail to get a kvtag from o%d\n", i + my_query_s); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) + printf("Total time to retrieve tags from %d objects: %.4f\n", n_query, total_time); + + fflush(stdout); + + for (i = 0; i < my_query; i++) { + if (*(int *)(values[i]) != i + my_add_tag_s) + printf("Error with retrieved tag from o%d\n", i + my_query_s); + // PDC_free_kvtag(&values[i]); + } + free(values); + + // close first object + for (i = 0; i < my_obj; i++) { + if (PDCobj_close(obj_ids[i]) < 0) + printf("fail to close object o%d\n", i + my_obj_s); + } + + // close a container + if (PDCcont_close(cont) < 0) + printf("fail to close container c1\n"); + + // close a container property + if (PDCprop_close(obj_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + if (PDCprop_close(cont_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + // close pdc + if (PDCclose(pdc) < 0) + printf("fail to close PDC\n"); + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return 0; +} From fb8cbf70e58e3f63d7f0f1bf9c2bf66b8067c906 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 23 Mar 2023 00:40:50 -0500 Subject: [PATCH 008/216] do while loop added, tested with 1m object and works --- src/tests/kvtag_scale_add_get.c | 144 ++++++++++++++++---------------- 1 file changed, 74 insertions(+), 70 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 1493a7d6d..4497392db 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -32,8 +32,9 @@ #include "pdc_client_connect.h" uint64_t -atoui64(char *arg) { - char *endptr; +atoui64(char *arg) +{ + char *endptr; uint64_t num = strtoull(arg, &endptr, 10); if (*endptr != '\0') { @@ -74,48 +75,43 @@ assign_work_to_rank(int rank, int size, uint64_t nwork, uint64_t *my_count, uint void print_usage(char *name) { - printf("%s n_obj n_add_tag n_query\n", name); + printf("%s n_obj n_obj_incr\n", name); } int main(int argc, char *argv[]) { - pdcid_t pdc, cont_prop, cont, obj_prop; - pdcid_t *obj_ids; - uint64_t n_obj, n_obj_incr, my_obj, my_obj_s; - uint64_t n_add_tag, my_add_tag, my_add_tag_s; - uint64_t n_query, my_query, my_query_s; - uint64_t i, v; - int proc_num, my_rank; - char obj_name[128]; - char tag_name[128]; - double stime, total_time; - pdc_kvtag_t kvtag; - void **values; - size_t value_size; + pdcid_t pdc, cont_prop, cont, obj_prop; + pdcid_t *obj_ids; + uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; + uint64_t i, v; + int proc_num, my_rank; + char obj_name[128]; + char tag_name[128]; + double stime, total_time; + void **values; + size_t value_size; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - if (argc != 5) { + if (argc != 3) { if (my_rank == 0) print_usage(argv[0]); goto done; } n_obj = atoui64(argv[1]); n_obj_incr = atoui64(argv[2]); - n_add_tag = atoui64(argv[3]); - n_query = atoui64(argv[4]); - if (n_add_tag > n_obj || n_query > n_obj) { + if (n_obj_incr > n_obj) { if (my_rank == 0) - printf("n_add_tag or n_query larger than n_obj! Exiting...\n"); + printf("n_obj_incr cannot be larger than n_obj! Exiting...\n"); goto done; } if (my_rank == 0) - printf("Create %d obj, %d tags, query %d\n", my_obj, my_add_tag, my_query); + printf("Create %llu obj, %llu tags, query %llu\n", my_obj, my_obj, my_obj); // create a pdc pdc = PDCinit("pdc"); @@ -135,75 +131,83 @@ main(int argc, char *argv[]) if (obj_prop <= 0) printf("Fail to create object property @ line %d!\n", __LINE__); - assign_work_to_rank(my_rank, proc_num, n_add_tag, &my_add_tag, &my_add_tag_s); - assign_work_to_rank(my_rank, proc_num, n_query, &my_query, &my_query_s); - assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); + curr_total_obj = 0; // Create a number of objects, add at least one tag to that object - obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); - for (i = 0; i < my_obj; i++) { - sprintf(obj_name, "obj%d", my_obj_s + i); - obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); - if (obj_ids[i] <= 0) - printf("Fail to create object @ line %d!\n", __LINE__); - } + obj_ids = (pdcid_t *)calloc(n_obj, sizeof(pdcid_t)); - if (my_rank == 0) - printf("Created %d objects\n", n_obj); + do { + assign_work_to_rank(my_rank, proc_num, n_obj_incr, &my_obj, &my_obj_s); + + for (i = 0; i < my_obj; i++) { + v = my_obj_s + i + curr_total_obj; + sprintf(obj_name, "obj%llu", v); + obj_ids[v] = PDCobj_create(cont, obj_name, obj_prop); + if (obj_ids[v] <= 0) + printf("Fail to create object @ line %d!\n", __LINE__); + } - // Add tags - kvtag.name = "Group"; - kvtag.value = (void *)&v; - kvtag.size = sizeof(uint64_t); + curr_total_obj += n_obj_incr; + + if (my_rank == 0) + printf("Created %llu objects in total now.\n", curr_total_obj); #ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - stime = MPI_Wtime(); + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); #endif - for (i = 0; i < my_add_tag; i++) { - v = i + my_add_tag_s; - if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) - printf("fail to add a kvtag to o%d\n", i + my_obj_s); - } + for (i = 0; i < my_obj; i++) { + v = i + my_obj_s + curr_total_obj - n_obj_incr; + sprintf(tag_name, "tag%llu", v); + if (PDCobj_put_tag(obj_ids[v], tag_name, (void *)&v, sizeof(uint64_t)) < 0) + printf("fail to add a kvtag to o%llu\n", i + my_obj_s); + } #ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - total_time = MPI_Wtime() - stime; + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; #endif - if (my_rank == 0) - printf("Total time to add tags to %d objects: %.4f\n", n_add_tag, total_time); + if (my_rank == 0) + printf("Total time to add tags to %llu objects: %.4f\n", my_obj, total_time); - values = (void **)calloc(my_query, sizeof(void *)); + values = (void **)calloc(my_obj, sizeof(void *)); #ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - stime = MPI_Wtime(); + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); #endif - for (i = 0; i < my_query; i++) { - if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_size) < 0) - printf("fail to get a kvtag from o%d\n", i + my_query_s); - } + for (i = 0; i < my_obj; i++) { + v = i + my_obj_s + curr_total_obj - n_obj_incr; + sprintf(tag_name, "tag%llu", v); + if (PDCobj_get_tag(obj_ids[v], tag_name, (void *)&values[i], (void *)&value_size) < 0) + printf("fail to get a kvtag from o%llu\n", v); + } #ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - total_time = MPI_Wtime() - stime; + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; #endif - if (my_rank == 0) - printf("Total time to retrieve tags from %d objects: %.4f\n", n_query, total_time); + if (my_rank == 0) + printf("Total time to retrieve tags from %llu objects: %.4f\n", my_obj, total_time); - fflush(stdout); + fflush(stdout); - for (i = 0; i < my_query; i++) { - if (*(int *)(values[i]) != i + my_add_tag_s) - printf("Error with retrieved tag from o%d\n", i + my_query_s); - // PDC_free_kvtag(&values[i]); - } - free(values); + for (i = 0; i < my_obj; i++) { + v = i + my_obj_s + curr_total_obj - n_obj_incr; + if (*(int *)(values[i]) != v) + printf("Error with retrieved tag from o%llu\n", v); + free(values[i]); + } + free(values); + + } while (curr_total_obj < n_obj); // close first object - for (i = 0; i < my_obj; i++) { - if (PDCobj_close(obj_ids[i]) < 0) - printf("fail to close object o%d\n", i + my_obj_s); + for (i = 0; i < n_obj; i++) { + if (obj_ids[i] > 0) { + if (PDCobj_close(obj_ids[i]) < 0) + printf("fail to close object o%llu\n", i); + } } // close a container From 5073c722b95b9c5e9c92a7128b8f3405710c95e4 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 23 Mar 2023 02:17:43 -0500 Subject: [PATCH 009/216] 1m objects test works, 10m object test fail as the original also fails --- src/tests/kvtag_scale_add_get.c | 109 +++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 30 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 4497392db..2215a6ab4 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -81,16 +81,18 @@ print_usage(char *name) int main(int argc, char *argv[]) { - pdcid_t pdc, cont_prop, cont, obj_prop; - pdcid_t *obj_ids; - uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; - uint64_t i, v; - int proc_num, my_rank; - char obj_name[128]; - char tag_name[128]; - double stime, total_time; - void **values; - size_t value_size; + pdcid_t pdc, cont_prop, cont, obj_prop; + pdcid_t *obj_ids; + uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; + uint64_t i, v; + int proc_num, my_rank; + char obj_name[128]; + char tag_name[128]; + double stime, total_time; + void **values; + size_t value_size; + obj_handle *oh; + struct pdc_obj_info *info; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); @@ -111,7 +113,7 @@ main(int argc, char *argv[]) } if (my_rank == 0) - printf("Create %llu obj, %llu tags, query %llu\n", my_obj, my_obj, my_obj); + printf("Create %llu obj, %llu tags, query %llu\n", n_obj, n_obj, n_obj); // create a pdc pdc = PDCinit("pdc"); @@ -133,33 +135,49 @@ main(int argc, char *argv[]) curr_total_obj = 0; + if (my_rank == 0) + printf("create obj_ids array\n"); // Create a number of objects, add at least one tag to that object - obj_ids = (pdcid_t *)calloc(n_obj, sizeof(pdcid_t)); + assign_work_to_rank(my_rank, proc_num, n_obj_incr, &my_obj, &my_obj_s); + obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + values = (void **)calloc(my_obj, sizeof(void *)); + sprintf(tag_name, "tag%d", 2); do { - assign_work_to_rank(my_rank, proc_num, n_obj_incr, &my_obj, &my_obj_s); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + + if (my_rank == 0) + printf("starting creating %llu objects... \n", my_obj); for (i = 0; i < my_obj; i++) { v = my_obj_s + i + curr_total_obj; sprintf(obj_name, "obj%llu", v); - obj_ids[v] = PDCobj_create(cont, obj_name, obj_prop); - if (obj_ids[v] <= 0) + obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); + if (obj_ids[i] <= 0) printf("Fail to create object @ line %d!\n", __LINE__); } curr_total_obj += n_obj_incr; +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif if (my_rank == 0) - printf("Created %llu objects in total now.\n", curr_total_obj); + printf("Created %llu objects in total now in %.4f seconds.\n", curr_total_obj, total_time); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif + for (i = 0; i < my_obj; i++) { v = i + my_obj_s + curr_total_obj - n_obj_incr; - sprintf(tag_name, "tag%llu", v); - if (PDCobj_put_tag(obj_ids[v], tag_name, (void *)&v, sizeof(uint64_t)) < 0) + if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)&v, sizeof(uint64_t)) < 0) printf("fail to add a kvtag to o%llu\n", i + my_obj_s); } @@ -170,16 +188,13 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("Total time to add tags to %llu objects: %.4f\n", my_obj, total_time); - values = (void **)calloc(my_obj, sizeof(void *)); - #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif for (i = 0; i < my_obj; i++) { v = i + my_obj_s + curr_total_obj - n_obj_incr; - sprintf(tag_name, "tag%llu", v); - if (PDCobj_get_tag(obj_ids[v], tag_name, (void *)&values[i], (void *)&value_size) < 0) + if (PDCobj_get_tag(obj_ids[i], tag_name, (void **)&values[i], (void *)&value_size) < 0) printf("fail to get a kvtag from o%llu\n", v); } @@ -196,19 +211,53 @@ main(int argc, char *argv[]) v = i + my_obj_s + curr_total_obj - n_obj_incr; if (*(int *)(values[i]) != v) printf("Error with retrieved tag from o%llu\n", v); - free(values[i]); + // free(values[i]); } - free(values); - - } while (curr_total_obj < n_obj); + // free(values); - // close first object - for (i = 0; i < n_obj; i++) { - if (obj_ids[i] > 0) { + // close objects + for (i = 0; i < my_obj; i++) { + v = i + my_obj_s + curr_total_obj - n_obj_incr; if (PDCobj_close(obj_ids[i]) < 0) - printf("fail to close object o%llu\n", i); + printf("fail to close object o%llu\n", v); } + + } while (curr_total_obj < n_obj); + + for (i = 0; i < my_obj; i++) { + free(values[i]); } + free(obj_ids); + free(values); + + // oh = PDCobj_iter_start(cont); + + // while (!PDCobj_iter_null(oh)) { + // info = PDCobj_iter_get_info(oh); + // info-> + // if (info->obj_pt->type != PDC_DOUBLE) { + // printf("Type is not properly inherited from object property.\n"); + // ret_value = 1; + // } + // if (info->obj_pt->ndim != ndim) { + // printf("Number of dimensions is not properly inherited from object property.\n"); + // ret_value = 1; + // } + // if (info->obj_pt->dims[0] != dims[0]) { + // printf("First dimension is not properly inherited from object property.\n"); + // ret_value = 1; + // } + // if (info->obj_pt->dims[1] != dims[1]) { + // printf("Second dimension is not properly inherited from object property.\n"); + // ret_value = 1; + // } + // if (info->obj_pt->dims[2] != dims[2]) { + // printf("Third dimension is not properly inherited from object property.\n"); + // ret_value = 1; + // } + + // oh = PDCobj_iter_next(oh, cont); + // } // close a container if (PDCcont_close(cont) < 0) From c73cb087d6ef3a695f17dbd027193844535e9313 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 23 Mar 2023 10:20:29 -0500 Subject: [PATCH 010/216] add new executable to test set --- src/tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index d0f319dd1..45eb40990 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -66,6 +66,7 @@ set(PROGRAMS kvtag_add_get_scale # kvtag_query # kvtag_query_scale + kvtag_scale_add_get # obj_transformation region_transfer_query region_transfer From d8d6b014e518bc9e5ca45a69255435658bb4110e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 23 Mar 2023 13:52:20 -0500 Subject: [PATCH 011/216] enlarge PDC_SERVER_ID_INTERVAL --- src/server/include/pdc_client_server_common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/include/pdc_client_server_common.h b/src/server/include/pdc_client_server_common.h index 11cb394c5..f30586833 100644 --- a/src/server/include/pdc_client_server_common.h +++ b/src/server/include/pdc_client_server_common.h @@ -69,7 +69,7 @@ hg_thread_mutex_t meta_obj_map_mutex_g; #define ADDR_MAX 512 #define DIM_MAX 4 #define TAG_LEN_MAX 2048 -#define PDC_SERVER_ID_INTERVEL 1000000 +#define PDC_SERVER_ID_INTERVEL 1000000000 #define PDC_SERVER_MAX_PROC_PER_NODE 32 #define PDC_SERIALIZE_MAX_SIZE 256 #define PDC_MAX_CORE_PER_NODE 68 // Cori KNL has 68 cores per node, Haswell 32 From fb74b5405276dababae3274be0eac6736918595a Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 24 Mar 2023 10:32:56 -0500 Subject: [PATCH 012/216] update code --- src/tests/kvtag_add_get_scale.c | 2 +- src/tests/kvtag_scale_add_get.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index b9a3a03ee..29d2b894a 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -81,7 +81,7 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - if (argc != 4) { + if (argc < 4) { if (my_rank == 0) print_usage(argv[0]); goto done; diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 2215a6ab4..d054a1a18 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -98,7 +98,7 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - if (argc != 3) { + if (argc < 3) { if (my_rank == 0) print_usage(argv[0]); goto done; From 3bc4ec11231bd1753c48911cb9e142ada639d5b7 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 24 Mar 2023 13:29:02 -0500 Subject: [PATCH 013/216] update console args --- src/tests/kvtag_query_scale.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/kvtag_query_scale.c b/src/tests/kvtag_query_scale.c index 58d276e55..80ca2a07c 100644 --- a/src/tests/kvtag_query_scale.c +++ b/src/tests/kvtag_query_scale.c @@ -70,7 +70,7 @@ main(int argc, char *argv[]) pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t * obj_ids; int n_obj, n_add_tag, my_obj, my_obj_s, my_add_tag, my_add_tag_s; - int proc_num, my_rank, i, v, iter; + int proc_num, my_rank, i, v, iter, round; char obj_name[128]; double stime, total_time; pdc_kvtag_t kvtag; @@ -83,14 +83,14 @@ main(int argc, char *argv[]) MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - if (argc != 2) { + if (argc < 3) { if (my_rank == 0) print_usage(argv[0]); goto done; } n_obj = atoi(argv[1]); + round = atoi(argv[2]); n_add_tag = n_obj / 100; - int round = 7; // create a pdc pdc = PDCinit("pdc"); From 61c26fb04fa7a3707cc13a690349ab67f9cd2744 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 24 Mar 2023 13:43:40 -0500 Subject: [PATCH 014/216] add p search test --- src/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 45eb40990..27cb1a3f3 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -65,7 +65,7 @@ set(PROGRAMS # kvtag_get kvtag_add_get_scale # kvtag_query -# kvtag_query_scale + kvtag_query_scale kvtag_scale_add_get # obj_transformation region_transfer_query From 74738344958be8092222232db392e640c524553d Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 30 Mar 2023 11:52:12 -0500 Subject: [PATCH 015/216] add console arg for changing number of attributes per object --- src/tests/kvtag_scale_add_get.c | 44 ++++++++------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index d054a1a18..6717928f5 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -84,7 +84,8 @@ main(int argc, char *argv[]) pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t *obj_ids; uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; - uint64_t i, v; + uint64_t n_attr; + uint64_t i, j, v; int proc_num, my_rank; char obj_name[128]; char tag_name[128]; @@ -98,13 +99,14 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - if (argc < 3) { + if (argc < 4) { if (my_rank == 0) print_usage(argv[0]); goto done; } n_obj = atoui64(argv[1]); n_obj_incr = atoui64(argv[2]); + n_attr = atoui64(argv[3]); if (n_obj_incr > n_obj) { if (my_rank == 0) @@ -193,9 +195,12 @@ main(int argc, char *argv[]) stime = MPI_Wtime(); #endif for (i = 0; i < my_obj; i++) { - v = i + my_obj_s + curr_total_obj - n_obj_incr; - if (PDCobj_get_tag(obj_ids[i], tag_name, (void **)&values[i], (void *)&value_size) < 0) - printf("fail to get a kvtag from o%llu\n", v); + for (j = 0; j < n_attr; j++) { + v = i + my_obj_s + curr_total_obj - n_obj_incr; + sprintf(tag_name, "tag%llu.%llu", j, v); + if (PDCobj_get_tag(obj_ids[i], tag_name, (void **)&values[i], (void *)&value_size) < 0) + printf("fail to get a kvtag from o%llu\n", v); + } } #ifdef ENABLE_MPI @@ -230,35 +235,6 @@ main(int argc, char *argv[]) free(obj_ids); free(values); - // oh = PDCobj_iter_start(cont); - - // while (!PDCobj_iter_null(oh)) { - // info = PDCobj_iter_get_info(oh); - // info-> - // if (info->obj_pt->type != PDC_DOUBLE) { - // printf("Type is not properly inherited from object property.\n"); - // ret_value = 1; - // } - // if (info->obj_pt->ndim != ndim) { - // printf("Number of dimensions is not properly inherited from object property.\n"); - // ret_value = 1; - // } - // if (info->obj_pt->dims[0] != dims[0]) { - // printf("First dimension is not properly inherited from object property.\n"); - // ret_value = 1; - // } - // if (info->obj_pt->dims[1] != dims[1]) { - // printf("Second dimension is not properly inherited from object property.\n"); - // ret_value = 1; - // } - // if (info->obj_pt->dims[2] != dims[2]) { - // printf("Third dimension is not properly inherited from object property.\n"); - // ret_value = 1; - // } - - // oh = PDCobj_iter_next(oh, cont); - // } - // close a container if (PDCcont_close(cont) < 0) printf("fail to close container c1\n"); From e6f811072a151f26520c82ed1fb55d1bbcd0ec1e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 30 Mar 2023 21:59:01 -0500 Subject: [PATCH 016/216] free allocated memory --- src/tests/kvtag_scale_add_get.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 6717928f5..c67aebaf7 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -143,15 +143,13 @@ main(int argc, char *argv[]) assign_work_to_rank(my_rank, proc_num, n_obj_incr, &my_obj, &my_obj_s); obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); - values = (void **)calloc(my_obj, sizeof(void *)); - sprintf(tag_name, "tag%d", 2); - do { + do { + values = (void **)calloc(my_obj, sizeof(void *)); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif - if (my_rank == 0) printf("starting creating %llu objects... \n", my_obj); @@ -216,9 +214,9 @@ main(int argc, char *argv[]) v = i + my_obj_s + curr_total_obj - n_obj_incr; if (*(int *)(values[i]) != v) printf("Error with retrieved tag from o%llu\n", v); - // free(values[i]); + free(values[i]); } - // free(values); + free(values); // close objects for (i = 0; i < my_obj; i++) { @@ -229,11 +227,12 @@ main(int argc, char *argv[]) } while (curr_total_obj < n_obj); - for (i = 0; i < my_obj; i++) { - free(values[i]); - } + // for (i = 0; i < my_obj; i++) { + // free(values[i]); + // } + // free(values); free(obj_ids); - free(values); + // close a container if (PDCcont_close(cont) < 0) From 9d3e73d53f9c9e917cc067ce411753f0a3979f80 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 31 Mar 2023 10:48:08 -0500 Subject: [PATCH 017/216] fix query count issue --- src/tests/kvtag_scale_add_get.c | 36 ++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index c67aebaf7..91424efb7 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -84,12 +84,13 @@ main(int argc, char *argv[]) pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t *obj_ids; uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; - uint64_t n_attr; - uint64_t i, j, v; - int proc_num, my_rank; + uint64_t n_attr, n_query; + uint64_t i, j, k, v; + int proc_num, my_rank, attr_value; char obj_name[128]; char tag_name[128]; double stime, total_time; + int *value_to_add; void **values; size_t value_size; obj_handle *oh; @@ -99,7 +100,7 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - if (argc < 4) { + if (argc < 6) { if (my_rank == 0) print_usage(argv[0]); goto done; @@ -107,6 +108,8 @@ main(int argc, char *argv[]) n_obj = atoui64(argv[1]); n_obj_incr = atoui64(argv[2]); n_attr = atoui64(argv[3]); + n_attr_len = atoui64(argv[4]); + n_query = atoui64(argv[5]); if (n_obj_incr > n_obj) { if (my_rank == 0) @@ -145,7 +148,7 @@ main(int argc, char *argv[]) obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); do { - values = (void **)calloc(my_obj, sizeof(void *)); + #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); @@ -177,8 +180,13 @@ main(int argc, char *argv[]) for (i = 0; i < my_obj; i++) { v = i + my_obj_s + curr_total_obj - n_obj_incr; - if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)&v, sizeof(uint64_t)) < 0) - printf("fail to add a kvtag to o%llu\n", i + my_obj_s); + for (j = 0; j < n_attr; j++) { + printf("print tag name before add"); + fflush(stdout); + sprintf(tag_name, "tag%llu.%llu", j, v); + if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)&v, sizeof(uint64_t)) < 0) + printf("fail to add a kvtag to o%llu\n", i + my_obj_s); + } } #ifdef ENABLE_MPI @@ -188,17 +196,18 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("Total time to add tags to %llu objects: %.4f\n", my_obj, total_time); + values = (void **)calloc(my_obj, sizeof(void *)); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif for (i = 0; i < my_obj; i++) { - for (j = 0; j < n_attr; j++) { - v = i + my_obj_s + curr_total_obj - n_obj_incr; - sprintf(tag_name, "tag%llu.%llu", j, v); - if (PDCobj_get_tag(obj_ids[i], tag_name, (void **)&values[i], (void *)&value_size) < 0) - printf("fail to get a kvtag from o%llu\n", v); - } + v = i + my_obj_s + curr_total_obj - n_obj_incr; + printf("print tag name before query"); + fflush(stdout); + sprintf(tag_name, "tag%llu.%llu", j, v); + if (PDCobj_get_tag(obj_ids[i], tag_name, (void **)&values[i], (void *)&value_size) < 0) + printf("fail to get a kvtag from o%llu\n", v); } #ifdef ENABLE_MPI @@ -232,7 +241,6 @@ main(int argc, char *argv[]) // } // free(values); free(obj_ids); - // close a container if (PDCcont_close(cont) < 0) From a1697ae5700429dc3d73f82736dbf7bec61629ad Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 31 Mar 2023 10:49:26 -0500 Subject: [PATCH 018/216] fix attr length definition --- src/tests/kvtag_scale_add_get.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 91424efb7..94e508650 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -84,7 +84,7 @@ main(int argc, char *argv[]) pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t *obj_ids; uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; - uint64_t n_attr, n_query; + uint64_t n_attr, n_attr_len, n_query; uint64_t i, j, k, v; int proc_num, my_rank, attr_value; char obj_name[128]; From d48a43981848f66394ef09e1f6baa21fab0e5799 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 31 Mar 2023 13:26:33 -0500 Subject: [PATCH 019/216] code refactored --- src/tests/kvtag_scale_add_get.c | 286 +++++++++++++++++++++++++------- 1 file changed, 222 insertions(+), 64 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 94e508650..a2ee774df 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -44,6 +44,22 @@ atoui64(char *arg) return num; } +/** + * Assigns a portion of work to a specific rank in a parallel processing environment. + * + * This function calculates the start and count of work items that should be + * assigned to a specific rank based on the total number of work items (nwork) and + * the total number of ranks (size) in the environment. + * + * @param rank The rank of the process for which the work is being assigned (0-indexed). + * @param size The total number of ranks in the parallel processing environment. + * @param nwork The total number of work items to be distributed among the ranks. + * @param my_count A pointer to a uint64_t variable that will store the number of work items assigned to the + * rank. + * @param my_start A pointer to a uint64_t variable that will store the starting index of the work items + * assigned to the rank. + * @return 0 if the function executes successfully, non-zero if an error occurs. + */ int assign_work_to_rank(int rank, int size, uint64_t nwork, uint64_t *my_count, uint64_t *my_start) { @@ -68,14 +84,195 @@ assign_work_to_rank(int rank, int size, uint64_t nwork, uint64_t *my_count, uint (*my_start) += (rank - (size - nwork % size)); } } + return 0; +} - return 1; +char ** +gen_strings(int n_strings, int string_len) +{ + srand(time(NULL)); // seed the random number generator with the current time + + char **str = malloc(n_strings * sizeof(char *)); // allocate memory for the array of strings + + for (int i = 0; i < n_strings; i++) { + str[i] = malloc((string_len + 1) * sizeof(char)); // allocate memory for each string in the array + for (int j = 0; j < string_len; j++) { + str[i][j] = 'a' + rand() % 26; // generate a random lowercase letter + } + str[i][string_len] = '\0'; // terminate the string with a null character + } + return str; // return the array of strings } +/** + * @brief Prints the usage instructions for the given program. + * This function displays the correct usage and command-line options for the program + * identified by the name parameter. It is typically called when the user provides + * incorrect or insufficient arguments. + * @param name A pointer to a character string representing the name of the program. + */ void print_usage(char *name) { - printf("%s n_obj n_obj_incr\n", name); + printf("Usage: %s \n\n", name); + + printf(" n_obj : The total number of objects (positive integer).\n"); + printf(" n_obj_incr : The increment in the number of objects per step (positive integer).\n"); + printf(" n_attr : The number of attributes per object (positive integer).\n"); + printf(" n_attr_len : The length of each attribute (positive integer).\n"); + printf(" n_query : The number of queries to be performed (positive integer).\n\n"); + + printf("Example:\n"); + printf(" %s 100 10 5 20 50\n\n", name); +} + +/** + * Initializes a test environment for the PDC by creating a specified number of objects in a container + * and returning their object IDs. + * + * @param my_rank The rank of the current process in the MPI communicator. + * @param proc_num The total number of processes in the MPI communicator. + * @param n_obj_incr Pointer to the number of objects to be created in each iteration. + * @param my_obj Pointer to the number of objects assigned to the current process. + * @param my_obj_s Pointer to the starting object index assigned to the current process. + * @param obj_prop Pointer to the object property ID to be used for creating objects. + * + * @return obj_ids Pointer to an array of object IDs for the created objects. + */ +pdcid_t * +init_test(int my_rank, int proc_num, uint64_t *n_obj_incr, uint64_t *my_obj, uint64_t *my_obj_s, + pdcid_t *obj_prop) +{ + // create a pdc + pdc = PDCinit("pdc"); + + // create a container property + cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); + + // create a container + cont = PDCcont_create("c1", cont_prop); + if (cont <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); + + // create an object property + *obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); + if (obj_prop <= 0) + printf("Fail to create object property @ line %d!\n", __LINE__); + + curr_total_obj = 0; + + if (my_rank == 0) + printf("create obj_ids array\n"); + // Create a number of objects, add at least one tag to that object + assign_work_to_rank(my_rank, proc_num, n_obj_incr, &my_obj, &my_obj_s); + + return (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); +} + +/** + * Creates a specified number of objects in a container and stores their object IDs in the obj_ids array. + * + * @param my_obj The number of objects assigned to the current process. + * @param my_obj_s The starting object index assigned to the current process. + * @param curr_total_obj The current total number of objects. + * @param cont The container ID in which to create the objects. + * @param obj_prop The object property ID to be used for creating objects. + * @param obj_ids Pointer to an array of object IDs for the created objects. + */ +void +create_object(uint64_t my_obj, uint64_t my_obj_s, uint64_t curr_total_obj, pdcid_t cont, pdcid_t obj_prop, + pdcid_t *obj_ids) +{ + uint64_t i, v; + char obj_name[128]; + + for (i = 0; i < my_obj; i++) { + v = my_obj_s + i + curr_total_obj; + sprintf(obj_name, "obj%llu", v); + obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); + if (obj_ids[i] <= 0) + printf("Fail to create object @ line %d!\n", __LINE__); + } +} + +/** + * Adds n_attr tags to each object in the obj_ids array. + * + * @param my_obj The number of objects assigned to the current process. + * @param my_obj_s The starting object index assigned to the current process. + * @param curr_total_obj The current total number of objects. + * @param n_obj_incr The number of objects to be created in each iteration. + * @param n_attr The number of attributes (tags) to add to each object. + * @param obj_ids Pointer to an array of object IDs for the objects to add tags. + */ +void +add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t curr_total_obj, uint64_t n_obj_incr, uint64_t n_attr, + char **tag_values, uint64_t tag_value_len, pdcid_t *obj_ids) +{ + uint64_t i, j, v; + char tag_name[128]; + + for (i = 0; i < my_obj; i++) { + v = i + my_obj_s + curr_total_obj - n_obj_incr; + for (j = 0; j < n_attr; j++) { + sprintf(tag_name, "tag%llu.%llu", v, j); + if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)&tag_values[j], tag_value_len * sizeof(char)) < + 0) + printf("fail to add a kvtag to o%llu\n", i + my_obj_s); + } + } +} + +/** + * Queries n_attr tags from the object specified by the object ID. + * + * @param obj_id The ID of the object to retrieve tags for. + * @param obj_name_v logical object ID for object name. + * @param n_attr The number of tags to retrieve. + * @param tag_values An array of pointers to store the tag values. + * @param value_size An array to store the size of each tag value. + */ +void +get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, int n_attr, char **tag_values, uint64_t *value_size) +{ + uint64_t i, v; + char tag_name[128]; + + for (i = 0; i < n_attr; i++) { + sprintf(tag_name, "tag%llu.%llu", obj_name_v, i); + if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], (void *)&value_size[i]) < 0) + printf("fail to get a kvtag from o%llu\n", v); + } +} + +/** + * Sends a specified number of queries to retrieve the values of the specified tags for a set of objects. + * + * @param my_obj_s The size of the current object name. + * @param curr_total_obj The current total number of objects. + * @param n_obj_incr The number of objects to increment by. + * @param n_query The number of queries to send. + * @param n_attr The number of tags to retrieve for each object. + * @param obj_ids An array of object IDs to retrieve tags for. + * @param tag_values An array of pointers to store the tag values for all queries and objects. + * The caller is responsible for allocating memory for the array and the individual + * pointers within it. + * @param value_size An array to store the size of each tag value. + * The caller is responsible for allocating memory for the array. + */ +void +send_queries(uint64_t my_obj_s, uint64_t curr_total_obj, uint64_t n_obj_incr, int n_query, uint64_t n_attr, + pdcid_t *obj_ids, char **tag_values, uint64_t *value_size) +{ + uint64_t i, v; + char tag_name[128]; + + for (i = 0; i < n_query; i++) { + v = i + my_obj_s + curr_total_obj - n_obj_incr; + get_object_tags(obj_ids[i], v, n_attr, &tag_values[i * n_attr], &value_size[i * n_attr]); + } } int @@ -91,8 +288,8 @@ main(int argc, char *argv[]) char tag_name[128]; double stime, total_time; int *value_to_add; - void **values; - size_t value_size; + char **query_rst_cache; + uint64_t *value_size; obj_handle *oh; struct pdc_obj_info *info; #ifdef ENABLE_MPI @@ -120,32 +317,8 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("Create %llu obj, %llu tags, query %llu\n", n_obj, n_obj, n_obj); - // create a pdc - pdc = PDCinit("pdc"); - - // create a container property - cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); - if (cont_prop <= 0) - printf("Fail to create container property @ line %d!\n", __LINE__); - - // create a container - cont = PDCcont_create("c1", cont_prop); - if (cont <= 0) - printf("Fail to create container @ line %d!\n", __LINE__); - - // create an object property - obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); - if (obj_prop <= 0) - printf("Fail to create object property @ line %d!\n", __LINE__); - - curr_total_obj = 0; - - if (my_rank == 0) - printf("create obj_ids array\n"); - // Create a number of objects, add at least one tag to that object - assign_work_to_rank(my_rank, proc_num, n_obj_incr, &my_obj, &my_obj_s); - - obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + // making necessary preparation for the test. + obj_ids = init_test(my_rank, proc_num, &n_obj_incr, &my_obj, &my_obj_s, &obj_prop); do { @@ -156,14 +329,10 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("starting creating %llu objects... \n", my_obj); - for (i = 0; i < my_obj; i++) { - v = my_obj_s + i + curr_total_obj; - sprintf(obj_name, "obj%llu", v); - obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); - if (obj_ids[i] <= 0) - printf("Fail to create object @ line %d!\n", __LINE__); - } - + // creating objects. Here, my_obj and my_obj_s has been calculated for each process based on + // n_obj_incr. + create_object(my_obj, my_obj_s, curr_total_obj, cont, obj_prop, obj_ids); + // therefore, after 'create_objects' function, we should add 'curr_total_obj' by 'n_obj_incr'. curr_total_obj += n_obj_incr; #ifdef ENABLE_MPI @@ -173,21 +342,14 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("Created %llu objects in total now in %.4f seconds.\n", curr_total_obj, total_time); + char **tag_values = gen_strings(n_attr, n_attr_len); + #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif - for (i = 0; i < my_obj; i++) { - v = i + my_obj_s + curr_total_obj - n_obj_incr; - for (j = 0; j < n_attr; j++) { - printf("print tag name before add"); - fflush(stdout); - sprintf(tag_name, "tag%llu.%llu", j, v); - if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)&v, sizeof(uint64_t)) < 0) - printf("fail to add a kvtag to o%llu\n", i + my_obj_s); - } - } + add_n_tags(my_obj, my_obj_s, curr_total_obj, n_obj_incr, n_attr, tag_values, obj_ids); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); @@ -196,19 +358,15 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("Total time to add tags to %llu objects: %.4f\n", my_obj, total_time); - values = (void **)calloc(my_obj, sizeof(void *)); + query_rst_cache = (char **)malloc(n_query * n_attr * sizeof(char *)); + value_size = malloc(n_query * n_attr * sizeof(uint64_t)); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif - for (i = 0; i < my_obj; i++) { - v = i + my_obj_s + curr_total_obj - n_obj_incr; - printf("print tag name before query"); - fflush(stdout); - sprintf(tag_name, "tag%llu.%llu", j, v); - if (PDCobj_get_tag(obj_ids[i], tag_name, (void **)&values[i], (void *)&value_size) < 0) - printf("fail to get a kvtag from o%llu\n", v); - } + + send_queries(my_obj_s, curr_total_obj, n_obj_incr, n_query, n_attr, obj_ids, query_rst_cache, + value_size); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); @@ -219,13 +377,13 @@ main(int argc, char *argv[]) fflush(stdout); - for (i = 0; i < my_obj; i++) { - v = i + my_obj_s + curr_total_obj - n_obj_incr; - if (*(int *)(values[i]) != v) - printf("Error with retrieved tag from o%llu\n", v); - free(values[i]); - } - free(values); + // for (i = 0; i < my_obj * n_attr; i++) { + // v = i + my_obj_s + curr_total_obj - n_obj_incr; + // if (*(int *)(values[i]) != v) + // printf("Error with retrieved tag from o%llu\n", v); + // free(values[i]); + // } + // free(values); // close objects for (i = 0; i < my_obj; i++) { From b5bdf6b351052736004d7fcb947a202f79a8dabd Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 31 Mar 2023 13:43:28 -0500 Subject: [PATCH 020/216] code refactored --- src/tests/kvtag_scale_add_get.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index a2ee774df..86d871326 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -143,6 +143,8 @@ pdcid_t * init_test(int my_rank, int proc_num, uint64_t *n_obj_incr, uint64_t *my_obj, uint64_t *my_obj_s, pdcid_t *obj_prop) { + + pdcid_t pdc, cont_prop, cont; // create a pdc pdc = PDCinit("pdc"); @@ -161,12 +163,11 @@ init_test(int my_rank, int proc_num, uint64_t *n_obj_incr, uint64_t *my_obj, uin if (obj_prop <= 0) printf("Fail to create object property @ line %d!\n", __LINE__); - curr_total_obj = 0; - if (my_rank == 0) printf("create obj_ids array\n"); + // Create a number of objects, add at least one tag to that object - assign_work_to_rank(my_rank, proc_num, n_obj_incr, &my_obj, &my_obj_s); + assign_work_to_rank(my_rank, proc_num, n_obj_incr, my_obj, my_obj_s); return (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); } @@ -278,7 +279,7 @@ send_queries(uint64_t my_obj_s, uint64_t curr_total_obj, uint64_t n_obj_incr, in int main(int argc, char *argv[]) { - pdcid_t pdc, cont_prop, cont, obj_prop; + pdcid_t obj_prop; pdcid_t *obj_ids; uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; uint64_t n_attr, n_attr_len, n_query; @@ -316,6 +317,10 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("Create %llu obj, %llu tags, query %llu\n", n_obj, n_obj, n_obj); + + curr_total_obj = 0; + + // making necessary preparation for the test. obj_ids = init_test(my_rank, proc_num, &n_obj_incr, &my_obj, &my_obj_s, &obj_prop); From 4e6a6ce7d9a8c4a2531d1d6e67c1f9facf0aca58 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 31 Mar 2023 13:50:23 -0500 Subject: [PATCH 021/216] code refactored --- src/tests/kvtag_scale_add_get.c | 53 +++++++++++++++++---------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 86d871326..286a59d4c 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -141,26 +141,24 @@ print_usage(char *name) */ pdcid_t * init_test(int my_rank, int proc_num, uint64_t *n_obj_incr, uint64_t *my_obj, uint64_t *my_obj_s, - pdcid_t *obj_prop) + pdcid_t *obj_prop, pdcid_t *pdc, pdcid_t *cont_prop, pdcid_t *cont) { - - pdcid_t pdc, cont_prop, cont; // create a pdc - pdc = PDCinit("pdc"); + *pdc = PDCinit("pdc"); // create a container property - cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); - if (cont_prop <= 0) + *cont_prop = PDCprop_create(PDC_CONT_CREATE, *pdc); + if (*cont_prop <= 0) printf("Fail to create container property @ line %d!\n", __LINE__); // create a container - cont = PDCcont_create("c1", cont_prop); - if (cont <= 0) + *cont = PDCcont_create("c1", *cont_prop); + if (*cont <= 0) printf("Fail to create container @ line %d!\n", __LINE__); // create an object property - *obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); - if (obj_prop <= 0) + *obj_prop = PDCprop_create(PDC_OBJ_CREATE, *pdc); + if (*obj_prop <= 0) printf("Fail to create object property @ line %d!\n", __LINE__); if (my_rank == 0) @@ -276,10 +274,27 @@ send_queries(uint64_t my_obj_s, uint64_t curr_total_obj, uint64_t n_obj_incr, in } } +void closePDC(pdcid_t pdc, pdcid_t cont_prop, pdcid_t cont, pdcid_t obj_prop){ + // close a container + if (PDCcont_close(cont) < 0) + printf("fail to close container c1\n"); + + // close a container property + if (PDCprop_close(obj_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + if (PDCprop_close(cont_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + // close pdc + if (PDCclose(pdc) < 0) + printf("fail to close PDC\n"); +} + int main(int argc, char *argv[]) { - pdcid_t obj_prop; + pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t *obj_ids; uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; uint64_t n_attr, n_attr_len, n_query; @@ -405,20 +420,8 @@ main(int argc, char *argv[]) // free(values); free(obj_ids); - // close a container - if (PDCcont_close(cont) < 0) - printf("fail to close container c1\n"); - - // close a container property - if (PDCprop_close(obj_prop) < 0) - printf("Fail to close property @ line %d\n", __LINE__); - - if (PDCprop_close(cont_prop) < 0) - printf("Fail to close property @ line %d\n", __LINE__); - - // close pdc - if (PDCclose(pdc) < 0) - printf("fail to close PDC\n"); + closePDC(pdc, cont_prop, cont, obj_prop); + done: #ifdef ENABLE_MPI From e98ed0f327144e321ca7ed4efe3aa36d1857be1f Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 31 Mar 2023 13:57:49 -0500 Subject: [PATCH 022/216] code refactored --- src/tests/kvtag_scale_add_get.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 286a59d4c..bfc806084 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -140,7 +140,7 @@ print_usage(char *name) * @return obj_ids Pointer to an array of object IDs for the created objects. */ pdcid_t * -init_test(int my_rank, int proc_num, uint64_t *n_obj_incr, uint64_t *my_obj, uint64_t *my_obj_s, +init_test(int my_rank, int proc_num, uint64_t n_obj_incr, uint64_t *my_obj, uint64_t *my_obj_s, pdcid_t *obj_prop, pdcid_t *pdc, pdcid_t *cont_prop, pdcid_t *cont) { // create a pdc @@ -167,7 +167,7 @@ init_test(int my_rank, int proc_num, uint64_t *n_obj_incr, uint64_t *my_obj, uin // Create a number of objects, add at least one tag to that object assign_work_to_rank(my_rank, proc_num, n_obj_incr, my_obj, my_obj_s); - return (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + return (pdcid_t *)calloc(*my_obj, sizeof(pdcid_t)); } /** @@ -338,7 +338,8 @@ main(int argc, char *argv[]) // making necessary preparation for the test. - obj_ids = init_test(my_rank, proc_num, &n_obj_incr, &my_obj, &my_obj_s, &obj_prop); + + obj_ids = init_test(my_rank, proc_num, n_obj_incr, &my_obj, &my_obj_s, &obj_prop, &pdc, &cont_prop, &cont); do { From 9e7f11ff9c73edb6d947d8f9551252efd5b0c679 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 31 Mar 2023 13:59:18 -0500 Subject: [PATCH 023/216] code refactored --- src/tests/kvtag_scale_add_get.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index bfc806084..a23f8eaa0 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -370,7 +370,7 @@ main(int argc, char *argv[]) stime = MPI_Wtime(); #endif - add_n_tags(my_obj, my_obj_s, curr_total_obj, n_obj_incr, n_attr, tag_values, obj_ids); + add_n_tags(my_obj, my_obj_s, curr_total_obj, n_obj_incr, n_attr, tag_values, n_attr_len, obj_ids); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); From d0e0b841ccac03d33f88a08ffddb02e7c80df2c3 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sat, 1 Apr 2023 18:21:01 -0500 Subject: [PATCH 024/216] code refactored --- src/tests/kvtag_scale_add_get.c | 118 ++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index a23f8eaa0..233900400 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -132,16 +132,20 @@ print_usage(char *name) * * @param my_rank The rank of the current process in the MPI communicator. * @param proc_num The total number of processes in the MPI communicator. - * @param n_obj_incr Pointer to the number of objects to be created in each iteration. + * @param n_obj_incr The number of objects to be created in each iteration. + * @param n_query The number of queries to be issued in each iteration. * @param my_obj Pointer to the number of objects assigned to the current process. * @param my_obj_s Pointer to the starting object index assigned to the current process. + * @param my_query Pointer to the number of queries assigned to the current process. + * @param my_query_s Pointer to the starting query index assigned to the current process. * @param obj_prop Pointer to the object property ID to be used for creating objects. * * @return obj_ids Pointer to an array of object IDs for the created objects. */ pdcid_t * -init_test(int my_rank, int proc_num, uint64_t n_obj_incr, uint64_t *my_obj, uint64_t *my_obj_s, - pdcid_t *obj_prop, pdcid_t *pdc, pdcid_t *cont_prop, pdcid_t *cont) +init_test(int my_rank, int proc_num, uint64_t n_obj_incr, uint64_t n_query, uint64_t *my_obj, + uint64_t *my_obj_s, uint64_t *my_query, uint64_t *my_query_s, pdcid_t *obj_prop, pdcid_t *pdc, + pdcid_t *cont_prop, pdcid_t *cont) { // create a pdc *pdc = PDCinit("pdc"); @@ -166,6 +170,7 @@ init_test(int my_rank, int proc_num, uint64_t n_obj_incr, uint64_t *my_obj, uint // Create a number of objects, add at least one tag to that object assign_work_to_rank(my_rank, proc_num, n_obj_incr, my_obj, my_obj_s); + assign_work_to_rank(my_rank, proc_num, n_query, my_query, my_query_s); return (pdcid_t *)calloc(*my_obj, sizeof(pdcid_t)); } @@ -175,20 +180,18 @@ init_test(int my_rank, int proc_num, uint64_t n_obj_incr, uint64_t *my_obj, uint * * @param my_obj The number of objects assigned to the current process. * @param my_obj_s The starting object index assigned to the current process. - * @param curr_total_obj The current total number of objects. * @param cont The container ID in which to create the objects. * @param obj_prop The object property ID to be used for creating objects. * @param obj_ids Pointer to an array of object IDs for the created objects. */ void -create_object(uint64_t my_obj, uint64_t my_obj_s, uint64_t curr_total_obj, pdcid_t cont, pdcid_t obj_prop, - pdcid_t *obj_ids) +create_object(uint64_t my_obj, uint64_t my_obj_s, pdcid_t cont, pdcid_t obj_prop, pdcid_t *obj_ids) { uint64_t i, v; char obj_name[128]; for (i = 0; i < my_obj; i++) { - v = my_obj_s + i + curr_total_obj; + v = i + my_obj_s; sprintf(obj_name, "obj%llu", v); obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); if (obj_ids[i] <= 0) @@ -201,20 +204,20 @@ create_object(uint64_t my_obj, uint64_t my_obj_s, uint64_t curr_total_obj, pdcid * * @param my_obj The number of objects assigned to the current process. * @param my_obj_s The starting object index assigned to the current process. - * @param curr_total_obj The current total number of objects. - * @param n_obj_incr The number of objects to be created in each iteration. * @param n_attr The number of attributes (tags) to add to each object. + * @param tag_values: Array of pointers to strings containing the tag values + * @param tag_value_len: 64-bit unsigned integer representing the length of the tag values in bytes * @param obj_ids Pointer to an array of object IDs for the objects to add tags. */ void -add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t curr_total_obj, uint64_t n_obj_incr, uint64_t n_attr, - char **tag_values, uint64_t tag_value_len, pdcid_t *obj_ids) +add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_values, uint64_t tag_value_len, + pdcid_t *obj_ids) { uint64_t i, j, v; char tag_name[128]; for (i = 0; i < my_obj; i++) { - v = i + my_obj_s + curr_total_obj - n_obj_incr; + v = i + my_obj_s; for (j = 0; j < n_attr; j++) { sprintf(tag_name, "tag%llu.%llu", v, j); if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)&tag_values[j], tag_value_len * sizeof(char)) < @@ -234,7 +237,7 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t curr_total_obj, uint64_t * @param value_size An array to store the size of each tag value. */ void -get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, int n_attr, char **tag_values, uint64_t *value_size) +get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, int n_attr, void **tag_values, uint64_t *value_size) { uint64_t i, v; char tag_name[128]; @@ -262,19 +265,44 @@ get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, int n_attr, char **tag_valu * The caller is responsible for allocating memory for the array. */ void -send_queries(uint64_t my_obj_s, uint64_t curr_total_obj, uint64_t n_obj_incr, int n_query, uint64_t n_attr, - pdcid_t *obj_ids, char **tag_values, uint64_t *value_size) +send_queries(uint64_t my_obj_s, int n_query, uint64_t n_attr, pdcid_t *obj_ids, void **tag_values, + uint64_t *value_size) { uint64_t i, v; - char tag_name[128]; for (i = 0; i < n_query; i++) { - v = i + my_obj_s + curr_total_obj - n_obj_incr; + v = i + my_obj_s; get_object_tags(obj_ids[i], v, n_attr, &tag_values[i * n_attr], &value_size[i * n_attr]); } } -void closePDC(pdcid_t pdc, pdcid_t cont_prop, pdcid_t cont, pdcid_t obj_prop){ +void +check_and_release_query_result(uint64_t n_query, uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_values, void **values, pdcid_t *obj_ids) +{ + uint64_t i, j, v; + + for (i = 0; i < n_query; i++) { + v = i + my_obj_s; + for (j = 0; j < n_attr; j++) { + char *query_rst = (char *)values[j + i * n_attr]; + if (strcmp(query_rst, tag_values[j])!=0) { + printf("Error with retrieved tag from o%llu. Expected %s, Found %s \n", v, tag_values[j], query_rst); + } + free(values[j + i * n_attr]); + } + } + free(values); + // close objects + for (i = 0; i < my_obj; i++) { + v = i + my_obj_s; + if (PDCobj_close(obj_ids[i]) < 0) + printf("fail to close object o%llu\n", v); + } +} + +void +closePDC(pdcid_t pdc, pdcid_t cont_prop, pdcid_t cont, pdcid_t obj_prop) +{ // close a container if (PDCcont_close(cont) < 0) printf("fail to close container c1\n"); @@ -297,7 +325,7 @@ main(int argc, char *argv[]) pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t *obj_ids; uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; - uint64_t n_attr, n_attr_len, n_query; + uint64_t n_attr, n_attr_len, n_query, my_query, my_query_s; uint64_t i, j, k, v; int proc_num, my_rank, attr_value; char obj_name[128]; @@ -332,14 +360,14 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("Create %llu obj, %llu tags, query %llu\n", n_obj, n_obj, n_obj); - + curr_total_obj = 0; + // making necessary preparation for the test. - + obj_ids = init_test(my_rank, proc_num, n_obj_incr, n_query, &my_obj, &my_obj_s, &my_query, &my_query_s, + &obj_prop, &pdc, &cont_prop, &cont); - // making necessary preparation for the test. - - obj_ids = init_test(my_rank, proc_num, n_obj_incr, &my_obj, &my_obj_s, &obj_prop, &pdc, &cont_prop, &cont); + char **tag_values = gen_strings(n_attr, n_attr_len); do { @@ -352,7 +380,7 @@ main(int argc, char *argv[]) // creating objects. Here, my_obj and my_obj_s has been calculated for each process based on // n_obj_incr. - create_object(my_obj, my_obj_s, curr_total_obj, cont, obj_prop, obj_ids); + create_object(my_obj, my_obj_s, cont, obj_prop, obj_ids); // therefore, after 'create_objects' function, we should add 'curr_total_obj' by 'n_obj_incr'. curr_total_obj += n_obj_incr; @@ -363,66 +391,50 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("Created %llu objects in total now in %.4f seconds.\n", curr_total_obj, total_time); - char **tag_values = gen_strings(n_attr, n_attr_len); - #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif - add_n_tags(my_obj, my_obj_s, curr_total_obj, n_obj_incr, n_attr, tag_values, n_attr_len, obj_ids); + add_n_tags(my_obj, my_obj_s, n_attr, tag_values, n_attr_len, obj_ids); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to add tags to %llu objects: %.4f\n", my_obj, total_time); + printf("Total time to add tags to %llu objects: %.4f\n", curr_total_obj, total_time); - query_rst_cache = (char **)malloc(n_query * n_attr * sizeof(char *)); - value_size = malloc(n_query * n_attr * sizeof(uint64_t)); + query_rst_cache = (void **)malloc(my_query * n_attr * sizeof(void *)); + value_size = malloc(my_query * n_attr * sizeof(uint64_t)); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif - send_queries(my_obj_s, curr_total_obj, n_obj_incr, n_query, n_attr, obj_ids, query_rst_cache, - value_size); + send_queries(my_obj_s, my_query, n_attr, obj_ids, query_rst_cache, value_size); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to retrieve tags from %llu objects: %.4f\n", my_obj, total_time); + printf("Total time to retrieve tags from %llu objects: %.4f\n", curr_total_obj, total_time); + check_and_release_query_result(my_query, my_obj, my_obj_s, n_attr, tag_values, query_rst_cache, obj_ids); fflush(stdout); - // for (i = 0; i < my_obj * n_attr; i++) { - // v = i + my_obj_s + curr_total_obj - n_obj_incr; - // if (*(int *)(values[i]) != v) - // printf("Error with retrieved tag from o%llu\n", v); - // free(values[i]); - // } - // free(values); - - // close objects - for (i = 0; i < my_obj; i++) { - v = i + my_obj_s + curr_total_obj - n_obj_incr; - if (PDCobj_close(obj_ids[i]) < 0) - printf("fail to close object o%llu\n", v); - } + my_obj_s += n_obj_incr; } while (curr_total_obj < n_obj); - // for (i = 0; i < my_obj; i++) { - // free(values[i]); - // } - // free(values); + for (i = 0; i < n_attr; i++) { + free(tag_values[i]); + } + free(tag_values); free(obj_ids); closePDC(pdc, cont_prop, cont, obj_prop); - done: #ifdef ENABLE_MPI From 6651d9207de7e581a6c71f22371e4b75c6935493 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sat, 1 Apr 2023 18:23:08 -0500 Subject: [PATCH 025/216] fix data type --- src/tests/kvtag_scale_add_get.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 233900400..dc5a413db 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -332,7 +332,7 @@ main(int argc, char *argv[]) char tag_name[128]; double stime, total_time; int *value_to_add; - char **query_rst_cache; + void **query_rst_cache; uint64_t *value_size; obj_handle *oh; struct pdc_obj_info *info; From 8963964d2c6df4bd08c8ecf1bbcd18cd3f4f3fe5 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 2 Apr 2023 08:22:23 -0500 Subject: [PATCH 026/216] fix data type --- src/tests/kvtag_scale_add_get.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index dc5a413db..e6c64a568 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -220,8 +220,7 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value v = i + my_obj_s; for (j = 0; j < n_attr; j++) { sprintf(tag_name, "tag%llu.%llu", v, j); - if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)&tag_values[j], tag_value_len * sizeof(char)) < - 0) + if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)&tag_values[j], tag_value_len + 1) < 0) printf("fail to add a kvtag to o%llu\n", i + my_obj_s); } } @@ -277,7 +276,8 @@ send_queries(uint64_t my_obj_s, int n_query, uint64_t n_attr, pdcid_t *obj_ids, } void -check_and_release_query_result(uint64_t n_query, uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_values, void **values, pdcid_t *obj_ids) +check_and_release_query_result(uint64_t n_query, uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, + char **tag_values, void **values, pdcid_t *obj_ids) { uint64_t i, j, v; @@ -285,8 +285,9 @@ check_and_release_query_result(uint64_t n_query, uint64_t my_obj, uint64_t my_ob v = i + my_obj_s; for (j = 0; j < n_attr; j++) { char *query_rst = (char *)values[j + i * n_attr]; - if (strcmp(query_rst, tag_values[j])!=0) { - printf("Error with retrieved tag from o%llu. Expected %s, Found %s \n", v, tag_values[j], query_rst); + if (strcmp(query_rst, tag_values[j]) != 0) { + printf("Error with retrieved tag from o%llu. Expected %s, Found %s \n", v, tag_values[j], + query_rst); } free(values[j + i * n_attr]); } @@ -421,7 +422,8 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("Total time to retrieve tags from %llu objects: %.4f\n", curr_total_obj, total_time); - check_and_release_query_result(my_query, my_obj, my_obj_s, n_attr, tag_values, query_rst_cache, obj_ids); + check_and_release_query_result(my_query, my_obj, my_obj_s, n_attr, tag_values, query_rst_cache, + obj_ids); fflush(stdout); my_obj_s += n_obj_incr; From 339d3e40870dc35f74a5682333513fd610c9eae9 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 2 Apr 2023 08:27:31 -0500 Subject: [PATCH 027/216] fix data type --- src/tests/kvtag_scale_add_get.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index e6c64a568..c9de1543c 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -220,7 +220,7 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value v = i + my_obj_s; for (j = 0; j < n_attr; j++) { sprintf(tag_name, "tag%llu.%llu", v, j); - if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)&tag_values[j], tag_value_len + 1) < 0) + if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], tag_value_len + 1) < 0) printf("fail to add a kvtag to o%llu\n", i + my_obj_s); } } From ff146cafd6011e9ce894654c8687e8d0523284b6 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 2 Apr 2023 09:10:13 -0500 Subject: [PATCH 028/216] add client side statistics --- src/tests/kvtag_scale_add_get.c | 39 ++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index c9de1543c..81087ebd7 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -327,11 +327,13 @@ main(int argc, char *argv[]) pdcid_t *obj_ids; uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; uint64_t n_attr, n_attr_len, n_query, my_query, my_query_s; + uint64_t n_servers, n_clients; uint64_t i, j, k, v; int proc_num, my_rank, attr_value; char obj_name[128]; char tag_name[128]; - double stime, total_time; + double stime, step_elapse, total_object_time, total_tag_time, total_query_time; + uint64_t total_object_count, total_tag_count, total_query_count; int *value_to_add; void **query_rst_cache; uint64_t *value_size; @@ -342,7 +344,7 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - if (argc < 6) { + if (argc < 7) { if (my_rank == 0) print_usage(argv[0]); goto done; @@ -352,6 +354,8 @@ main(int argc, char *argv[]) n_attr = atoui64(argv[3]); n_attr_len = atoui64(argv[4]); n_query = atoui64(argv[5]); + n_servers = atoui64(argv[6]); + n_clients = proc_num; if (n_obj_incr > n_obj) { if (my_rank == 0) @@ -370,6 +374,9 @@ main(int argc, char *argv[]) char **tag_values = gen_strings(n_attr, n_attr_len); + total_time = 0; + k = 1; + do { #ifdef ENABLE_MPI @@ -387,10 +394,13 @@ main(int argc, char *argv[]) #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); - total_time = MPI_Wtime() - stime; + step_elapse = MPI_Wtime() - stime; + total_object_time += step_elapse; + total_object_count += n_obj_incr; #endif if (my_rank == 0) - printf("Created %llu objects in total now in %.4f seconds.\n", curr_total_obj, total_time); + printf("Iteration %llu : Objects: %llu , Time: %.4f sec. Object throughput in this iteration: %.4f .\n", k, n_obj_incr, step_elapse, ((double)n_obj_incr)/step_elapse); + printf("Overall %llu : Objects: %llu , Time: %.4f sec. Overall object throughput: %.4f .\n", k, total_object_count, total_object_time, ((double)total_object_count)/total_object_time); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); @@ -401,10 +411,13 @@ main(int argc, char *argv[]) #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); - total_time = MPI_Wtime() - stime; + step_elapse = MPI_Wtime() - stime; + total_tag_time += step_elapse; + total_tag_count += n_obj_incr * n_attr; #endif if (my_rank == 0) - printf("Total time to add tags to %llu objects: %.4f\n", curr_total_obj, total_time); + printf("Iteration %llu : Tags: %llu , Time: %.4f sec. Tag throughput in this iteration: %.4f .\n", k, n_obj_incr * n_attr, step_elapse, (double)(n_obj_incr * n_attr)/step_elapse); + printf("Overall %llu : Tags: %llu , Time: %.4f sec. Overall tag throughput: %.4f .\n", k, total_tag_count, total_tag_time, ((double)total_tag_count)/total_tag_time); query_rst_cache = (void **)malloc(my_query * n_attr * sizeof(void *)); value_size = malloc(my_query * n_attr * sizeof(uint64_t)); @@ -417,19 +430,29 @@ main(int argc, char *argv[]) #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); - total_time = MPI_Wtime() - stime; + step_elapse = MPI_Wtime() - stime; + total_query_time += step_elapse; + total_query_count += n_query * n_attr; #endif if (my_rank == 0) - printf("Total time to retrieve tags from %llu objects: %.4f\n", curr_total_obj, total_time); + printf("Iteration %llu : Queries: %llu , Time: %.4f sec. Query throughput in this iteration: %.4f .\n", k, n_query * n_attr, step_elapse, (double)(n_query * n_attr)/step_elapse); + printf("Overall %llu : Queries: %llu , Time: %.4f sec. Overall query throughput: %.4f .\n", k, total_query_count, total_query_time, ((double)total_query_count)/total_query_time); check_and_release_query_result(my_query, my_obj, my_obj_s, n_attr, tag_values, query_rst_cache, obj_ids); fflush(stdout); my_obj_s += n_obj_incr; + k++; } while (curr_total_obj < n_obj); + if (my_rank == 0): + printf("Final Report: \n"); + printf("Servers: %llu , Clients: %llu , C/S ratio: %.4f \n", n_servers, n_clients, (double)n_clients/(double)n_servers); + printf("Iterations: %llu , Objects: %llu , Tags/Object: %llu , Queries/Iteration: %llu , \n", k, curr_total_obj, n_attr, n_query); + printf("Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", (double)curr_total_obj/total_object_time, (double)(curr_total_obj*n_attr)/total_tag_time, (double)(total_query_count * n_attr)/total_query_time); + for (i = 0; i < n_attr; i++) { free(tag_values[i]); } From 33e3fdf6f8e55fac2fb8c26e8171da0532d91b24 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 2 Apr 2023 09:18:44 -0500 Subject: [PATCH 029/216] add client side statistics --- src/tests/kvtag_scale_add_get.c | 47 +++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 81087ebd7..01a9abcc3 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -325,15 +325,15 @@ main(int argc, char *argv[]) { pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t *obj_ids; - uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj; + uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj=0; uint64_t n_attr, n_attr_len, n_query, my_query, my_query_s; uint64_t n_servers, n_clients; uint64_t i, j, k, v; int proc_num, my_rank, attr_value; char obj_name[128]; char tag_name[128]; - double stime, step_elapse, total_object_time, total_tag_time, total_query_time; - uint64_t total_object_count, total_tag_count, total_query_count; + double stime=0.0, step_elapse=0.0, total_object_time=0.0, total_tag_time=0.0, total_query_time=0.0; + uint64_t total_object_count=0, total_tag_count=0, total_query_count=0; int *value_to_add; void **query_rst_cache; uint64_t *value_size; @@ -366,7 +366,6 @@ main(int argc, char *argv[]) if (my_rank == 0) printf("Create %llu obj, %llu tags, query %llu\n", n_obj, n_obj, n_obj); - curr_total_obj = 0; // making necessary preparation for the test. obj_ids = init_test(my_rank, proc_num, n_obj_incr, n_query, &my_obj, &my_obj_s, &my_query, &my_query_s, @@ -374,8 +373,7 @@ main(int argc, char *argv[]) char **tag_values = gen_strings(n_attr, n_attr_len); - total_time = 0; - k = 1; + k = 1; do { @@ -399,8 +397,12 @@ main(int argc, char *argv[]) total_object_count += n_obj_incr; #endif if (my_rank == 0) - printf("Iteration %llu : Objects: %llu , Time: %.4f sec. Object throughput in this iteration: %.4f .\n", k, n_obj_incr, step_elapse, ((double)n_obj_incr)/step_elapse); - printf("Overall %llu : Objects: %llu , Time: %.4f sec. Overall object throughput: %.4f .\n", k, total_object_count, total_object_time, ((double)total_object_count)/total_object_time); + printf("Iteration %llu : Objects: %llu , Time: %.4f sec. Object throughput in this iteration: " + "%.4f .\n", + k, n_obj_incr, step_elapse, ((double)n_obj_incr) / step_elapse); + printf( + "Overall %llu : Objects: %llu , Time: %.4f sec. Overall object throughput: %.4f .\n", + k, total_object_count, total_object_time, ((double)total_object_count) / total_object_time); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); @@ -416,8 +418,10 @@ main(int argc, char *argv[]) total_tag_count += n_obj_incr * n_attr; #endif if (my_rank == 0) - printf("Iteration %llu : Tags: %llu , Time: %.4f sec. Tag throughput in this iteration: %.4f .\n", k, n_obj_incr * n_attr, step_elapse, (double)(n_obj_incr * n_attr)/step_elapse); - printf("Overall %llu : Tags: %llu , Time: %.4f sec. Overall tag throughput: %.4f .\n", k, total_tag_count, total_tag_time, ((double)total_tag_count)/total_tag_time); + printf("Iteration %llu : Tags: %llu , Time: %.4f sec. Tag throughput in this iteration: %.4f .\n", + k, n_obj_incr * n_attr, step_elapse, (double)(n_obj_incr * n_attr) / step_elapse); + printf("Overall %llu : Tags: %llu , Time: %.4f sec. Overall tag throughput: %.4f .\n", k, + total_tag_count, total_tag_time, ((double)total_tag_count) / total_tag_time); query_rst_cache = (void **)malloc(my_query * n_attr * sizeof(void *)); value_size = malloc(my_query * n_attr * sizeof(uint64_t)); @@ -435,8 +439,12 @@ main(int argc, char *argv[]) total_query_count += n_query * n_attr; #endif if (my_rank == 0) - printf("Iteration %llu : Queries: %llu , Time: %.4f sec. Query throughput in this iteration: %.4f .\n", k, n_query * n_attr, step_elapse, (double)(n_query * n_attr)/step_elapse); - printf("Overall %llu : Queries: %llu , Time: %.4f sec. Overall query throughput: %.4f .\n", k, total_query_count, total_query_time, ((double)total_query_count)/total_query_time); + printf("Iteration %llu : Queries: %llu , Time: %.4f sec. Query throughput in this iteration: " + "%.4f .\n", + k, n_query * n_attr, step_elapse, (double)(n_query * n_attr) / step_elapse); + printf( + "Overall %llu : Queries: %llu , Time: %.4f sec. Overall query throughput: %.4f .\n", + k, total_query_count, total_query_time, ((double)total_query_count) / total_query_time); check_and_release_query_result(my_query, my_obj, my_obj_s, n_attr, tag_values, query_rst_cache, obj_ids); @@ -447,12 +455,17 @@ main(int argc, char *argv[]) } while (curr_total_obj < n_obj); - if (my_rank == 0): + if (my_rank == 0) { printf("Final Report: \n"); - printf("Servers: %llu , Clients: %llu , C/S ratio: %.4f \n", n_servers, n_clients, (double)n_clients/(double)n_servers); - printf("Iterations: %llu , Objects: %llu , Tags/Object: %llu , Queries/Iteration: %llu , \n", k, curr_total_obj, n_attr, n_query); - printf("Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", (double)curr_total_obj/total_object_time, (double)(curr_total_obj*n_attr)/total_tag_time, (double)(total_query_count * n_attr)/total_query_time); - + printf("Servers: %llu , Clients: %llu , C/S ratio: %.4f \n", n_servers, n_clients, + (double)n_clients / (double)n_servers); + printf("Iterations: %llu , Objects: %llu , Tags/Object: %llu , Queries/Iteration: %llu , \n", k, + curr_total_obj, n_attr, n_query); + printf("Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", + (double)curr_total_obj / total_object_time, (double)(curr_total_obj * n_attr) / total_tag_time, + (double)(total_query_count * n_attr) / total_query_time); + } + for (i = 0; i < n_attr; i++) { free(tag_values[i]); } From 150a7369dde017fce41970cff7ed8c8e28f6b5a3 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 2 Apr 2023 09:25:01 -0500 Subject: [PATCH 030/216] fix format --- src/tests/kvtag_scale_add_get.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 01a9abcc3..5d5e52f0e 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -396,7 +396,7 @@ main(int argc, char *argv[]) total_object_time += step_elapse; total_object_count += n_obj_incr; #endif - if (my_rank == 0) + if (my_rank == 0){ printf("Iteration %llu : Objects: %llu , Time: %.4f sec. Object throughput in this iteration: " "%.4f .\n", k, n_obj_incr, step_elapse, ((double)n_obj_incr) / step_elapse); @@ -404,6 +404,8 @@ main(int argc, char *argv[]) "Overall %llu : Objects: %llu , Time: %.4f sec. Overall object throughput: %.4f .\n", k, total_object_count, total_object_time, ((double)total_object_count) / total_object_time); + } + #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); @@ -417,11 +419,13 @@ main(int argc, char *argv[]) total_tag_time += step_elapse; total_tag_count += n_obj_incr * n_attr; #endif - if (my_rank == 0) + if (my_rank == 0){ printf("Iteration %llu : Tags: %llu , Time: %.4f sec. Tag throughput in this iteration: %.4f .\n", k, n_obj_incr * n_attr, step_elapse, (double)(n_obj_incr * n_attr) / step_elapse); printf("Overall %llu : Tags: %llu , Time: %.4f sec. Overall tag throughput: %.4f .\n", k, total_tag_count, total_tag_time, ((double)total_tag_count) / total_tag_time); + } + query_rst_cache = (void **)malloc(my_query * n_attr * sizeof(void *)); value_size = malloc(my_query * n_attr * sizeof(uint64_t)); @@ -438,13 +442,14 @@ main(int argc, char *argv[]) total_query_time += step_elapse; total_query_count += n_query * n_attr; #endif - if (my_rank == 0) + if (my_rank == 0) { printf("Iteration %llu : Queries: %llu , Time: %.4f sec. Query throughput in this iteration: " "%.4f .\n", k, n_query * n_attr, step_elapse, (double)(n_query * n_attr) / step_elapse); printf( "Overall %llu : Queries: %llu , Time: %.4f sec. Overall query throughput: %.4f .\n", k, total_query_count, total_query_time, ((double)total_query_count) / total_query_time); + } check_and_release_query_result(my_query, my_obj, my_obj_s, n_attr, tag_values, query_rst_cache, obj_ids); From 1e6d53bf1584e7f2e3d6c02d8d1e038a982d2072 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 3 Apr 2023 09:44:18 -0500 Subject: [PATCH 031/216] clang formatter --- src/tests/kvtag_scale_add_get.c | 56 ++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 5d5e52f0e..00060319a 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -325,15 +325,16 @@ main(int argc, char *argv[]) { pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t *obj_ids; - uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj=0; + uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj = 0; uint64_t n_attr, n_attr_len, n_query, my_query, my_query_s; uint64_t n_servers, n_clients; uint64_t i, j, k, v; int proc_num, my_rank, attr_value; char obj_name[128]; char tag_name[128]; - double stime=0.0, step_elapse=0.0, total_object_time=0.0, total_tag_time=0.0, total_query_time=0.0; - uint64_t total_object_count=0, total_tag_count=0, total_query_count=0; + double stime = 0.0, step_elapse = 0.0; + double total_object_time = 0.0, total_tag_time = 0.0, total_query_time = 0.0; + uint64_t total_object_count = 0, total_tag_count = 0, total_query_count = 0; int *value_to_add; void **query_rst_cache; uint64_t *value_size; @@ -373,7 +374,7 @@ main(int argc, char *argv[]) char **tag_values = gen_strings(n_attr, n_attr_len); - k = 1; + k = 1; do { @@ -381,9 +382,6 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif - if (my_rank == 0) - printf("starting creating %llu objects... \n", my_obj); - // creating objects. Here, my_obj and my_obj_s has been calculated for each process based on // n_obj_incr. create_object(my_obj, my_obj_s, cont, obj_prop, obj_ids); @@ -396,16 +394,16 @@ main(int argc, char *argv[]) total_object_time += step_elapse; total_object_count += n_obj_incr; #endif - if (my_rank == 0){ - printf("Iteration %llu : Objects: %llu , Time: %.4f sec. Object throughput in this iteration: " + if (my_rank == 0) { + printf("Iteration %4llu : Objects: %6llu , Time: %.4f sec. Object throughput in this iteration: " "%.4f .\n", k, n_obj_incr, step_elapse, ((double)n_obj_incr) / step_elapse); - printf( - "Overall %llu : Objects: %llu , Time: %.4f sec. Overall object throughput: %.4f .\n", - k, total_object_count, total_object_time, ((double)total_object_count) / total_object_time); - + printf("Overall %4llu : Objects: %6llu , Time: %.4f sec. Overall object throughput: " + "%.4f .\n", + k, total_object_count, total_object_time, + ((double)total_object_count) / total_object_time); } - + #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); @@ -419,13 +417,14 @@ main(int argc, char *argv[]) total_tag_time += step_elapse; total_tag_count += n_obj_incr * n_attr; #endif - if (my_rank == 0){ - printf("Iteration %llu : Tags: %llu , Time: %.4f sec. Tag throughput in this iteration: %.4f .\n", - k, n_obj_incr * n_attr, step_elapse, (double)(n_obj_incr * n_attr) / step_elapse); - printf("Overall %llu : Tags: %llu , Time: %.4f sec. Overall tag throughput: %.4f .\n", k, - total_tag_count, total_tag_time, ((double)total_tag_count) / total_tag_time); + if (my_rank == 0) { + printf( + "Iteration %4llu : Tags: %6llu , Time: %.4f sec. Tag throughput in this iteration: %.4f .\n", + k, n_obj_incr * n_attr, step_elapse, (double)(n_obj_incr * n_attr) / step_elapse); + printf( + "Overall %4llu : Tags: %6llu , Time: %.4f sec. Overall tag throughput: %.4f .\n", + k, total_tag_count, total_tag_time, ((double)total_tag_count) / total_tag_time); } - query_rst_cache = (void **)malloc(my_query * n_attr * sizeof(void *)); value_size = malloc(my_query * n_attr * sizeof(uint64_t)); @@ -443,12 +442,12 @@ main(int argc, char *argv[]) total_query_count += n_query * n_attr; #endif if (my_rank == 0) { - printf("Iteration %llu : Queries: %llu , Time: %.4f sec. Query throughput in this iteration: " + printf("Iteration %4llu : Queries: %6llu , Time: %.4f sec. Query throughput in this iteration: " "%.4f .\n", k, n_query * n_attr, step_elapse, (double)(n_query * n_attr) / step_elapse); - printf( - "Overall %llu : Queries: %llu , Time: %.4f sec. Overall query throughput: %.4f .\n", - k, total_query_count, total_query_time, ((double)total_query_count) / total_query_time); + printf("Overall %4llu : Queries: %6llu , Time: %.4f sec. Overall query throughput: " + "%.4f .\n", + k, total_query_count, total_query_time, ((double)total_query_count) / total_query_time); } check_and_release_query_result(my_query, my_obj, my_obj_s, n_attr, tag_values, query_rst_cache, @@ -462,11 +461,12 @@ main(int argc, char *argv[]) if (my_rank == 0) { printf("Final Report: \n"); - printf("Servers: %llu , Clients: %llu , C/S ratio: %.4f \n", n_servers, n_clients, + printf("[Final Report 1] Servers: %llu , Clients: %llu , C/S ratio: %.4f \n", n_servers, n_clients, (double)n_clients / (double)n_servers); - printf("Iterations: %llu , Objects: %llu , Tags/Object: %llu , Queries/Iteration: %llu , \n", k, - curr_total_obj, n_attr, n_query); - printf("Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", + printf("[Final Report 2] Iterations: %llu , Objects: %llu , Tags/Object: %llu , Queries/Iteration: " + "%llu , \n", + k, curr_total_obj, n_attr, n_query); + printf("[Final Report 3] Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", (double)curr_total_obj / total_object_time, (double)(curr_total_obj * n_attr) / total_tag_time, (double)(total_query_count * n_attr) / total_query_time); } From 84e7675a58bf4050c682ab89a46002fcdf609f2e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 3 Apr 2023 10:01:53 -0500 Subject: [PATCH 032/216] update CMake --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 447ef9924..e54244a42 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -433,6 +433,8 @@ configure_file( ${PDC_BINARY_DIR}/CMakeFiles/pdc-config-version.cmake @ONLY ) + + install( FILES ${PDC_BINARY_DIR}/CMakeFiles/pdc-config-version.cmake @@ -444,10 +446,14 @@ install( DIRECTORY ${PDC_BINARY_DIR}/bin + FILES_MATCHING PATTERN "*" DESTINATION ${PDC_INSTALL_DATA_DIR}/test ) +# set(ADD_EXE_PERMISSION_CMD "chmod +x ${}/test/*") +# add_custom_command(TARGET ${PROJECT_NAME} POST_INSTALL COMMAND ${add_permission_cmd}) + #install( # FILES From 54a9419df00a8ec89617935c3bfd5ea0466e5bd3 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 3 Apr 2023 10:04:24 -0500 Subject: [PATCH 033/216] update CMake --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e54244a42..0633ac6bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -445,10 +445,10 @@ install( install( DIRECTORY ${PDC_BINARY_DIR}/bin - - FILES_MATCHING PATTERN "*" DESTINATION ${PDC_INSTALL_DATA_DIR}/test + FILES_MATCHING PATTERN "*" + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) # set(ADD_EXE_PERMISSION_CMD "chmod +x ${}/test/*") From 3a8f63e329301ceb154a6f73df71151cc0896082 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 3 Apr 2023 10:08:32 -0500 Subject: [PATCH 034/216] update CMake --- CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0633ac6bd..7a1db4d08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -451,10 +451,6 @@ install( PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) -# set(ADD_EXE_PERMISSION_CMD "chmod +x ${}/test/*") -# add_custom_command(TARGET ${PROJECT_NAME} POST_INSTALL COMMAND ${add_permission_cmd}) - - #install( # FILES # ${PDC_BINARY_DIR}/bin/pdc_server.exe From dfa1e50c08a7ff856979809d0fb35e1e095d4b49 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 3 Apr 2023 10:24:35 -0500 Subject: [PATCH 035/216] free allocated memory properly --- src/tests/kvtag_add_get_scale.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index 29d2b894a..95a32591a 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -178,7 +178,7 @@ main(int argc, char *argv[]) for (i = 0; i < my_query; i++) { if (*(int *)(values[i]) != i + my_add_tag_s) printf("Error with retrieved tag from o%d\n", i + my_query_s); - // PDC_free_kvtag(&values[i]); + free(values[i]); } free(values); From c6c0be6fc7149ea3a89eb0217a6a8d28e991f00e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 3 Apr 2023 13:42:47 -0500 Subject: [PATCH 036/216] clang format --- src/api/pdc_client_connect.c | 2 +- src/tests/kvtag_scale_add_get.c | 12 ++++++------ src/utils/pdc_region_utils.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index 83cba609f..22e98f751 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -1195,7 +1195,7 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po init_info.na_init_info.progress_mode = NA_NO_BLOCK; // busy mode #endif -//#ifndef PDC_HAS_CRAY_DRC +// #ifndef PDC_HAS_CRAY_DRC #ifdef PDC_HAS_SHARED_SERVER init_info.auto_sm = HG_TRUE; #endif diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_scale_add_get.c index 00060319a..5586f4b6c 100644 --- a/src/tests/kvtag_scale_add_get.c +++ b/src/tests/kvtag_scale_add_get.c @@ -34,7 +34,7 @@ uint64_t atoui64(char *arg) { - char *endptr; + char * endptr; uint64_t num = strtoull(arg, &endptr, 10); if (*endptr != '\0') { @@ -324,7 +324,7 @@ int main(int argc, char *argv[]) { pdcid_t pdc, cont_prop, cont, obj_prop; - pdcid_t *obj_ids; + pdcid_t * obj_ids; uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj = 0; uint64_t n_attr, n_attr_len, n_query, my_query, my_query_s; uint64_t n_servers, n_clients; @@ -335,10 +335,10 @@ main(int argc, char *argv[]) double stime = 0.0, step_elapse = 0.0; double total_object_time = 0.0, total_tag_time = 0.0, total_query_time = 0.0; uint64_t total_object_count = 0, total_tag_count = 0, total_query_count = 0; - int *value_to_add; - void **query_rst_cache; - uint64_t *value_size; - obj_handle *oh; + int * value_to_add; + void ** query_rst_cache; + uint64_t * value_size; + obj_handle * oh; struct pdc_obj_info *info; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); diff --git a/src/utils/pdc_region_utils.c b/src/utils/pdc_region_utils.c index a81dd9b9f..3f360056e 100644 --- a/src/utils/pdc_region_utils.c +++ b/src/utils/pdc_region_utils.c @@ -33,7 +33,7 @@ PDC_region_overlap_detect(int ndim, uint64_t *offset1, uint64_t *size1, uint64_t for (i = 0; i < ndim; ++i) { output_offset[0][i] = offset2[i] < offset1[i] ? offset1[i] : offset2[i]; output_size[0][i] = ((offset2[i] + size2[i] < offset1[i] + size1[i]) ? (offset2[i] + size2[i]) - : (offset1[i] + size1[i])) - + : (offset1[i] + size1[i])) - output_offset[0][i]; } From cb6f440324273cf06174a6fece38d46151818468 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 3 Apr 2023 15:13:41 -0500 Subject: [PATCH 037/216] clang format --- .github/workflows/clang-format-check.yml | 2 +- .github/workflows/clang-format-fix.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml index 52e4d0613..7f90f5fb2 100644 --- a/.github/workflows/clang-format-check.yml +++ b/.github/workflows/clang-format-check.yml @@ -13,6 +13,6 @@ jobs: with: source: '.' extensions: 'c,h,cpp,hpp' - clangFormatVersion: 10 + clangFormatVersion: 11 style: file # exclude: './config' diff --git a/.github/workflows/clang-format-fix.yml b/.github/workflows/clang-format-fix.yml index 9145f65ca..ac3592185 100644 --- a/.github/workflows/clang-format-fix.yml +++ b/.github/workflows/clang-format-fix.yml @@ -14,7 +14,7 @@ jobs: with: source: '.' extensions: 'c,h,cpp,hpp' - clangFormatVersion: 10 + clangFormatVersion: 11 inplace: True style: file # exclude: './config ' From 74d72670c4a02642d40cb643ddec6a8ae0b2561e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 3 Apr 2023 15:32:44 -0500 Subject: [PATCH 038/216] clang-format-10 --- .github/workflows/clang-format-check.yml | 2 +- .github/workflows/clang-format-fix.yml | 2 +- src/utils/pdc_region_utils.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml index 7f90f5fb2..52e4d0613 100644 --- a/.github/workflows/clang-format-check.yml +++ b/.github/workflows/clang-format-check.yml @@ -13,6 +13,6 @@ jobs: with: source: '.' extensions: 'c,h,cpp,hpp' - clangFormatVersion: 11 + clangFormatVersion: 10 style: file # exclude: './config' diff --git a/.github/workflows/clang-format-fix.yml b/.github/workflows/clang-format-fix.yml index ac3592185..9145f65ca 100644 --- a/.github/workflows/clang-format-fix.yml +++ b/.github/workflows/clang-format-fix.yml @@ -14,7 +14,7 @@ jobs: with: source: '.' extensions: 'c,h,cpp,hpp' - clangFormatVersion: 11 + clangFormatVersion: 10 inplace: True style: file # exclude: './config ' diff --git a/src/utils/pdc_region_utils.c b/src/utils/pdc_region_utils.c index 3f360056e..a81dd9b9f 100644 --- a/src/utils/pdc_region_utils.c +++ b/src/utils/pdc_region_utils.c @@ -33,7 +33,7 @@ PDC_region_overlap_detect(int ndim, uint64_t *offset1, uint64_t *size1, uint64_t for (i = 0; i < ndim; ++i) { output_offset[0][i] = offset2[i] < offset1[i] ? offset1[i] : offset2[i]; output_size[0][i] = ((offset2[i] + size2[i] < offset1[i] + size1[i]) ? (offset2[i] + size2[i]) - : (offset1[i] + size1[i])) - + : (offset1[i] + size1[i])) - output_offset[0][i]; } From 25196ed9dc350936b1d2ea12890842b982fb2227 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 3 Apr 2023 15:38:17 -0500 Subject: [PATCH 039/216] change file name --- src/tests/CMakeLists.txt | 1 + src/tests/{kvtag_scale_add_get.c => kvtag_add_get_benchmark.c} | 0 2 files changed, 1 insertion(+) rename src/tests/{kvtag_scale_add_get.c => kvtag_add_get_benchmark.c} (100%) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 27cb1a3f3..e524881e8 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -63,6 +63,7 @@ set(PROGRAMS # data_server_meta_test kvtag_add_get # kvtag_get + kvtag_add_get_benchmark kvtag_add_get_scale # kvtag_query kvtag_query_scale diff --git a/src/tests/kvtag_scale_add_get.c b/src/tests/kvtag_add_get_benchmark.c similarity index 100% rename from src/tests/kvtag_scale_add_get.c rename to src/tests/kvtag_add_get_benchmark.c From 50d10014b42b997811d8e55dfa1bad4619d7ea86 Mon Sep 17 00:00:00 2001 From: zhangwei217245 Date: Mon, 3 Apr 2023 20:05:19 -0700 Subject: [PATCH 040/216] address review comments --- src/tests/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index e524881e8..499421880 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -67,7 +67,6 @@ set(PROGRAMS kvtag_add_get_scale # kvtag_query kvtag_query_scale - kvtag_scale_add_get # obj_transformation region_transfer_query region_transfer From 8cdccdaa25091fc81f73552ba11523defe82829f Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 4 Apr 2023 13:35:49 -0500 Subject: [PATCH 041/216] update llsm importer --- src/server/pdc_server.c | 4 ++-- tools/CMakeLists.txt | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 9b5428a5c..526658bd0 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -61,8 +61,8 @@ #include #endif -#define PDC_CHECKPOINT_INTERVAL 200 -#define PDC_CHECKPOINT_MIN_INTERVAL_SEC 300 +#define PDC_CHECKPOINT_INTERVAL 1000 +#define PDC_CHECKPOINT_MIN_INTERVAL_SEC 1800 // Global debug variable to control debug printfs int is_debug_g = 0; diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index f9915b221..e2a017bd4 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -51,8 +51,21 @@ option(USE_SYSTEM_HDF5 "Use system-installed HDF5." ON) endif() endif() +option(USE_SYSTEM_OPENMP "Use system-installed OpenMP." ON) +if(USE_SYSTEM_OPENMP) + find_package(OpenMP REQUIRED) + + if(OPENMP_FOUND) + set(OPENMP_LIBRARIES "${OpenMP_C_LIBRARIES}") + else() + message(FATAL_ERROR "OpenMP not found") + endif() +endif() + + add_definitions(-DENABLE_MPI=1) add_library(cjson cjson/cJSON.c) +add_subdirectory(llsm) set(PROGRAMS pdc_ls From 6ebb5a7fc54d81821c93982c3c096b2a0d8db979 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 4 Apr 2023 13:37:13 -0500 Subject: [PATCH 042/216] update llsm importer --- tools/llsm/CMakeLists.txt | 5 + tools/llsm/parallelReadTiff.c | 762 ++++++++++++++++++++++++++++++++++ tools/llsm/parallelReadTiff.h | 0 tools/llsm_importer.c | 14 + 4 files changed, 781 insertions(+) create mode 100644 tools/llsm/CMakeLists.txt create mode 100644 tools/llsm/parallelReadTiff.c create mode 100644 tools/llsm/parallelReadTiff.h create mode 100644 tools/llsm_importer.c diff --git a/tools/llsm/CMakeLists.txt b/tools/llsm/CMakeLists.txt new file mode 100644 index 000000000..a0ea9404d --- /dev/null +++ b/tools/llsm/CMakeLists.txt @@ -0,0 +1,5 @@ + +add_library(llsm_tiff parallelReadTiff.c) +target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS}) +target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES}) \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c new file mode 100644 index 000000000..fb4283e6a --- /dev/null +++ b/tools/llsm/parallelReadTiff.c @@ -0,0 +1,762 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "tiffio.h" +#include "omp.h" +#include "mex.h" +//mex -v COPTIMFLAGS="-O3 -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-I/global/home/groups/software/sl-7.x86_64/modules/libtiff/4.1.0/libtiff/' '-L/global/home/groups/software/sl-7.x86_64/modules/libtiff/4.1.0/libtiff/' -ltiff parallelReadTiff.c +//mex COMPFLAGS='$COMPFLAGS /openmp' '-IC:\Program Files (x86)\tiff\include\' '-LC:\Program Files (x86)\tiff\lib\' -ltiffd.lib C:\Users\Matt\Documents\parallelTiff\main.cpp + +//libtiff 4.4.0 +//mex -v COPTIMFLAGS="-O3 -DNDEBUG" LDOPTIMFLAGS="-O3 -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-I/clusterfs/fiona/matthewmueller/software/tiff-4.4.0/include' '-L/clusterfs/fiona/matthewmueller/software/tiff-4.4.0/lib' -ltiff parallelReadTiff.c + +// Handle the tilde character in filenames on Linux/Mac +#ifndef _WIN32 +#include +char* expandTilde(char* path) { + wordexp_t expPath; + wordexp(path, &expPath, 0); + return expPath.we_wordv[0]; +} +#endif + +void DummyHandler(const char* module, const char* fmt, va_list ap) +{ + // ignore errors and warnings +} + +// Backup method in case there are errors reading strips +void readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char* fileName, void* tiff, uint64_t bits, uint64_t startSlice, uint8_t flipXY){ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (z-1)/numWorkers+1; + uint64_t bytes = bits/8; + + int32_t w; + #pragma omp parallel for + for(w = 0; w < numWorkers; w++){ + + TIFF* tif = TIFFOpen(fileName, "r"); + if(!tif) mexErrMsgIdAndTxt("tiff:threadError","Thread %d: File \"%s\" cannot be opened\n",w,fileName); + + void* buffer = malloc(x*bytes); + for(int64_t dir = startSlice+(w*batchSize); dir < startSlice+((w+1)*batchSize); dir++){ + if(dir>=z+startSlice) break; + + int counter = 0; + while(!TIFFSetDirectory(tif, (uint64_t)dir) && counter<3){ + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n",w,fileName,dir,counter+1); + counter++; + } + + for (int64_t i = 0; i < y; i++) + { + TIFFReadScanline(tif, buffer, i, 0); + if(!flipXY){ + memcpy(tiff+((i*x)*bytes),buffer,x*bytes); + continue; + } + //loading the data into a buffer + switch(bits){ + case 8: + // Map Values to flip x and y for MATLAB + for(int64_t j = 0; j < x; j++){ + ((uint8_t*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((uint8_t*)buffer)[j]; + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for(int64_t j = 0; j < x; j++){ + ((uint16_t*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((uint16_t*)buffer)[j]; + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for(int64_t j = 0; j < x; j++){ + ((float*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((float*)buffer)[j]; + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for(int64_t j = 0; j < x; j++){ + ((double*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((double*)buffer)[j]; + } + break; + } + } + } + free(buffer); + TIFFClose(tif); + } +} + +void readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char* fileName, void* tiff, uint64_t bits, uint64_t startSlice, uint64_t stripSize, uint8_t flipXY){ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (z-1)/numWorkers+1; + uint64_t bytes = bits/8; + + uint16_t compressed = 1; + TIFF* tif = TIFFOpen(fileName, "r"); + TIFFGetField(tif, TIFFTAG_COMPRESSION, &compressed); + + + + + int32_t w; + uint8_t errBak = 0; + uint8_t err = 0; + char errString[10000]; + if(compressed > 1 || z < 32768){ + TIFFClose(tif); + #pragma omp parallel for + for(w = 0; w < numWorkers; w++){ + + uint8_t outCounter = 0; + TIFF* tif = TIFFOpen(fileName, "r"); + while(!tif){ + tif = TIFFOpen(fileName, "r"); + if(outCounter == 3){ + #pragma omp critical + { + err = 1; + sprintf(errString,"Thread %d: File \"%s\" cannot be opened\n",w,fileName); + } + continue; + } + outCounter++; + } + + void* buffer = malloc(x*stripSize*bytes); + for(int64_t dir = startSlice+(w*batchSize); dir < startSlice+((w+1)*batchSize); dir++){ + if(dir>=z+startSlice || err) break; + + uint8_t counter = 0; + while(!TIFFSetDirectory(tif, (uint64_t)dir) && counter<3){ + counter++; + if(counter == 3){ + #pragma omp critical + { + err = 1; + sprintf(errString,"Thread %d: File \"%s\" cannot be opened\n",w,fileName); + } + } + } + if(err) break; + for (int64_t i = 0; i*stripSize < y; i++) + { + + //loading the data into a buffer + int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize*x*bytes); + if(cBytes < 0){ + #pragma omp critical + { + errBak = 1; + err = 1; + sprintf(errString,"Thread %d: Strip %ld cannot be read\n",w,i); + } + break; + } + if(!flipXY){ + memcpy(tiff+((i*stripSize*x)*bytes),buffer,cBytes); + continue; + } + switch(bits){ + case 8: + // Map Values to flip x and y for MATLAB + for(int64_t k = 0; k < stripSize; k++){ + if((k+(i*stripSize)) >= y) break; + for(int64_t j = 0; j < x; j++){ + ((uint8_t*)tiff)[((j*y)+(k+(i*stripSize)))+((dir-startSlice)*(x*y))] = ((uint8_t*)buffer)[j+(k*x)]; + } + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for(int64_t k = 0; k < stripSize; k++){ + if((k+(i*stripSize)) >= y) break; + for(int64_t j = 0; j < x; j++){ + ((uint16_t*)tiff)[((j*y)+(k+(i*stripSize)))+((dir-startSlice)*(x*y))] = ((uint16_t*)buffer)[j+(k*x)]; + } + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for(int64_t k = 0; k < stripSize; k++){ + if((k+(i*stripSize)) >= y) break; + for(int64_t j = 0; j < x; j++){ + ((float*)tiff)[((j*y)+(k+(i*stripSize)))+((dir-startSlice)*(x*y))] = ((float*)buffer)[j+(k*x)]; + } + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for(int64_t k = 0; k < stripSize; k++){ + if((k+(i*stripSize)) >= y) break; + for(int64_t j = 0; j < x; j++){ + ((double*)tiff)[((j*y)+(k+(i*stripSize)))+((dir-startSlice)*(x*y))] = ((double*)buffer)[j+(k*x)]; + } + } + break; + } + } + } + free(buffer); + TIFFClose(tif); + } + } + else{ + uint64_t stripsPerDir = (uint64_t)ceil((double)y/(double)stripSize); + #ifdef _WIN32 + int fd = open(fileName,O_RDONLY | O_BINARY); + #else + int fd = open(fileName,O_RDONLY); + #endif + if(fd == -1) mexErrMsgIdAndTxt("disk:threadError","File \"%s\" cannot be opened from Disk\n",fileName); + + if(!tif) mexErrMsgIdAndTxt("tiff:threadError","File \"%s\" cannot be opened\n",fileName); + uint64_t offset = 0; + uint64_t* offsets = NULL; + TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); + uint64_t* byteCounts = NULL; + TIFFGetField(tif, TIFFTAG_STRIPBYTECOUNTS, &byteCounts); + if(!offsets || !byteCounts) mexErrMsgIdAndTxt("tiff:threadError","Could not get offsets or byte counts from the tiff file\n"); + offset = offsets[0]; + uint64_t fOffset = offsets[stripsPerDir-1]+byteCounts[stripsPerDir-1]; + uint64_t zSize = fOffset-offset; + TIFFSetDirectory(tif,1); + TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); + uint64_t gap = offsets[0]-fOffset; + + lseek(fd, offset, SEEK_SET); + + + TIFFClose(tif); + uint64_t curr = 0; + uint64_t bytesRead = 0; + // TESTING + // Not sure if we will need to read in chunks like for ImageJ + for(uint64_t i = 0; i < z; i++){ + bytesRead = read(fd,tiff+curr,zSize); + curr += bytesRead; + lseek(fd,gap,SEEK_CUR); + } + close(fd); + uint64_t size = x*y*z*(bits/8); + void* tiffC = malloc(size); + memcpy(tiffC,tiff,size); + #pragma omp parallel for + for(uint64_t k = 0; k < z; k++){ + for(uint64_t j = 0; j < x; j++){ + for(uint64_t i = 0; i < y; i++){ + switch(bits){ + case 8: + ((uint8_t*)tiff)[i+(j*y)+(k*x*y)] = ((uint8_t*)tiffC)[j+(i*x)+(k*x*y)]; + break; + case 16: + ((uint16_t*)tiff)[i+(j*y)+(k*x*y)] = ((uint16_t*)tiffC)[j+(i*x)+(k*x*y)]; + break; + case 32: + ((float*)tiff)[i+(j*y)+(k*x*y)] = ((float*)tiffC)[j+(i*x)+(k*x*y)]; + break; + case 64: + ((double*)tiff)[i+(j*y)+(k*x*y)] = ((double*)tiffC)[j+(i*x)+(k*x*y)]; + break; + } + } + } + } + free(tiffC); + } + if(err){ + if(errBak) readTiffParallelBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); + else mexErrMsgIdAndTxt("tiff:threadError",errString); + } +} + +// Backup method in case there are errors reading strips +void readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char* fileName, void* tiff, uint64_t bits, uint64_t startSlice, uint8_t flipXY){ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (y-1)/numWorkers+1; + uint64_t bytes = bits/8; + + int32_t w; + #pragma omp parallel for + for(w = 0; w < numWorkers; w++){ + + TIFF* tif = TIFFOpen(fileName, "r"); + if(!tif) mexErrMsgIdAndTxt("tiff:threadError","Thread %d: File \"%s\" cannot be opened\n",w,fileName); + + void* buffer = malloc(x*bytes); + for(int64_t dir = startSlice+(w*batchSize); dir < startSlice+((w+1)*batchSize); dir++){ + if(dir>=z+startSlice) break; + + int counter = 0; + while(!TIFFSetDirectory(tif, (uint64_t)0) && counter<3){ + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n",w,fileName,dir,counter+1); + counter++; + } + + for (int64_t i = (w*batchSize); i < ((w+1)*batchSize); i++) + { + if(i >= y) break; + TIFFReadScanline(tif, buffer, i, 0); + if(!flipXY){ + memcpy(tiff+((i*x)*bytes),buffer,x*bytes); + continue; + } + //loading the data into a buffer + switch(bits){ + case 8: + // Map Values to flip x and y for MATLAB + for(int64_t j = 0; j < x; j++){ + ((uint8_t*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((uint8_t*)buffer)[j]; + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for(int64_t j = 0; j < x; j++){ + ((uint16_t*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((uint16_t*)buffer)[j]; + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for(int64_t j = 0; j < x; j++){ + ((float*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((float*)buffer)[j]; + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for(int64_t j = 0; j < x; j++){ + ((double*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((double*)buffer)[j]; + } + break; + } + } + } + free(buffer); + TIFFClose(tif); + } +} + +void readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char* fileName, void* tiff, uint64_t bits, uint64_t startSlice, uint64_t stripSize, uint8_t flipXY){ + int32_t numWorkers = omp_get_max_threads(); + uint64_t stripsPerDir = (uint64_t)ceil((double)y/(double)stripSize); + int32_t batchSize = (stripsPerDir-1)/numWorkers+1; + uint64_t bytes = bits/8; + + int32_t w; + uint8_t err = 0; + uint8_t errBak = 0; + char errString[10000]; + + + #pragma omp parallel for + for(w = 0; w < numWorkers; w++){ + + uint8_t outCounter = 0; + TIFF* tif = TIFFOpen(fileName, "r"); + while(!tif){ + tif = TIFFOpen(fileName, "r"); + if(outCounter == 3){ + #pragma omp critical + { + err = 1; + sprintf(errString,"Thread %d: File \"%s\" cannot be opened\n",w,fileName); + } + continue; + } + outCounter++; + } + + void* buffer = malloc(x*stripSize*bytes); + + + uint8_t counter = 0; + while(!TIFFSetDirectory(tif, 0) && counter<3){ + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n",w,fileName,0,counter+1); + counter++; + if(counter == 3){ + #pragma omp critical + { + err = 1; + sprintf(errString,"Thread %d: File \"%s\" cannot be opened\n",w,fileName); + } + } + } + for (int64_t i = (w*batchSize); i < (w+1)*batchSize; i++) + { + if(i*stripSize >= y || err) break; + //loading the data into a buffer + int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize*x*bytes); + if(cBytes < 0){ + #pragma omp critical + { + errBak = 1; + err = 1; + sprintf(errString,"Thread %d: Strip %ld cannot be read\n",w,i); + } + break; + } + if(!flipXY){ + memcpy(tiff+((i*stripSize*x)*bytes),buffer,cBytes); + continue; + } + switch(bits){ + case 8: + // Map Values to flip x and y for MATLAB + for(int64_t k = 0; k < stripSize; k++){ + if((k+(i*stripSize)) >= y) break; + for(int64_t j = 0; j < x; j++){ + ((uint8_t*)tiff)[((j*y)+(k+(i*stripSize)))] = ((uint8_t*)buffer)[j+(k*x)]; + } + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for(int64_t k = 0; k < stripSize; k++){ + if((k+(i*stripSize)) >= y) break; + for(int64_t j = 0; j < x; j++){ + ((uint16_t*)tiff)[((j*y)+(k+(i*stripSize)))] = ((uint16_t*)buffer)[j+(k*x)]; + } + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for(int64_t k = 0; k < stripSize; k++){ + if((k+(i*stripSize)) >= y) break; + for(int64_t j = 0; j < x; j++){ + ((float*)tiff)[((j*y)+(k+(i*stripSize)))] = ((float*)buffer)[j+(k*x)]; + } + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for(int64_t k = 0; k < stripSize; k++){ + if((k+(i*stripSize)) >= y) break; + for(int64_t j = 0; j < x; j++){ + ((double*)tiff)[((j*y)+(k+(i*stripSize)))] = ((double*)buffer)[j+(k*x)]; + } + } + break; + } + } + free(buffer); + TIFFClose(tif); + } + + if(err) { + if(errBak) readTiffParallel2DBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); + else mexErrMsgIdAndTxt("tiff:threadError",errString); + } +} + +// Reading images saved by ImageJ +void readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char* fileName, void* tiff, uint64_t bits, uint64_t startSlice, uint64_t stripSize, uint8_t flipXY){ + #ifdef _WIN32 + int fd = open(fileName,O_RDONLY | O_BINARY); + #else + int fd = open(fileName,O_RDONLY); + #endif + TIFF* tif = TIFFOpen(fileName, "r"); + if(!tif) mexErrMsgIdAndTxt("tiff:threadError","File \"%s\" cannot be opened\n",fileName); + uint64_t offset = 0; + uint64_t* offsets = NULL; + TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); + if(offsets) offset = offsets[0]; + + TIFFClose(tif); + lseek(fd, offset, SEEK_SET); + uint64_t bytes = bits/8; + //#pragma omp parallel for + /* + for(uint64_t i = 0; i < z; i++){ + uint64_t cOffset = x*y*bytes*i; + //pread(fd,tiff+cOffset,x*y*bytes,offset+cOffset); + read(fd,tiff+cOffset,x*y*bytes); + }*/ + uint64_t chunk = 0; + uint64_t tBytes = x*y*z*bytes; + uint64_t bytesRead; + uint64_t rBytes = tBytes; + if(tBytes < INT_MAX) bytesRead = read(fd,tiff,tBytes); + else{ + while(chunk < tBytes){ + rBytes = tBytes-chunk; + if(rBytes > INT_MAX) bytesRead = read(fd,tiff+chunk,INT_MAX); + else bytesRead = read(fd,tiff+chunk,rBytes); + chunk += bytesRead; + } + } + close(fd); + // Swap endianess for types greater than 8 bits + // TODO: May need to change later because we may not always need to swap + if(bits > 8){ + #pragma omp parallel for + for(uint64_t i = 0; i < x*y*z; i++){ + switch(bits){ + case 16: + //((uint16_t*)tiff)[i] = ((((uint16_t*)tiff)[i] & 0xff) >> 8) | (((uint16_t*)tiff)[i] << 8); + //((uint16_t*)tiff)[i] = bswap_16(((uint16_t*)tiff)[i]); + ((uint16_t*)tiff)[i] = ((((uint16_t*)tiff)[i] << 8) & 0xff00) | ((((uint16_t*)tiff)[i] >> 8) & 0x00ff); + break; + case 32: + //((num & 0xff000000) >> 24) | ((num & 0x00ff0000) >> 8) | ((num & 0x0000ff00) << 8) | (num << 24) + //((float*)tiff)[i] = bswap_32(((float*)tiff)[i]); + ((uint32_t*)tiff)[i] = ((((uint32_t*)tiff)[i] << 24) & 0xff000000 ) | + ((((uint32_t*)tiff)[i] << 8) & 0x00ff0000 ) | + ((((uint32_t*)tiff)[i] >> 8) & 0x0000ff00 ) | + ((((uint32_t*)tiff)[i] >> 24) & 0x000000ff ); + break; + case 64: + //((double*)tiff)[i] = bswap_64(((double*)tiff)[i]); + ((uint64_t*)tiff)[i] = ( (((uint64_t*)tiff)[i] << 56) & 0xff00000000000000UL ) | + ( (((uint64_t*)tiff)[i] << 40) & 0x00ff000000000000UL ) | + ( (((uint64_t*)tiff)[i] << 24) & 0x0000ff0000000000UL ) | + ( (((uint64_t*)tiff)[i] << 8) & 0x000000ff00000000UL ) | + ( (((uint64_t*)tiff)[i] >> 8) & 0x00000000ff000000UL ) | + ( (((uint64_t*)tiff)[i] >> 24) & 0x0000000000ff0000UL ) | + ( (((uint64_t*)tiff)[i] >> 40) & 0x000000000000ff00UL ) | + ( (((uint64_t*)tiff)[i] >> 56) & 0x00000000000000ffUL ); + break; + } + + } + } + // Find a way to do this in-place without making a copy + if(flipXY){ + uint64_t size = x*y*z*(bits/8); + void* tiffC = malloc(size); + memcpy(tiffC,tiff,size); + #pragma omp parallel for + for(uint64_t k = 0; k < z; k++){ + for(uint64_t j = 0; j < x; j++){ + for(uint64_t i = 0; i < y; i++){ + switch(bits){ + case 8: + ((uint8_t*)tiff)[i+(j*y)+(k*x*y)] = ((uint8_t*)tiffC)[j+(i*x)+(k*x*y)]; + break; + case 16: + ((uint16_t*)tiff)[i+(j*y)+(k*x*y)] = ((uint16_t*)tiffC)[j+(i*x)+(k*x*y)]; + break; + case 32: + ((float*)tiff)[i+(j*y)+(k*x*y)] = ((float*)tiffC)[j+(i*x)+(k*x*y)]; + break; + case 64: + ((double*)tiff)[i+(j*y)+(k*x*y)] = ((double*)tiffC)[j+(i*x)+(k*x*y)]; + break; + } + } + } + } + free(tiffC); + } +} + +uint8_t isImageJIm(const char* fileName){ + TIFF* tif = TIFFOpen(fileName, "r"); + if(!tif) return 0; + char* tiffDesc = NULL; + if(TIFFGetField(tif, TIFFTAG_IMAGEDESCRIPTION, &tiffDesc)){ + if(strstr(tiffDesc, "ImageJ")){ + return 1; + } + } + return 0; +} + +uint64_t imageJImGetZ(const char* fileName){ + TIFF* tif = TIFFOpen(fileName, "r"); + if(!tif) return 0; + char* tiffDesc = NULL; + if(TIFFGetField(tif, TIFFTAG_IMAGEDESCRIPTION, &tiffDesc)){ + if(strstr(tiffDesc, "ImageJ")){ + char* nZ = strstr(tiffDesc,"images="); + if(nZ){ + nZ+=7; + char* temp; + return strtol(nZ,&temp,10); + } + } + } + return 0; +} + +void mexFunction(int nlhs, mxArray *plhs[], + int nrhs, const mxArray *prhs[]) +{ + // Check if the fileName is a char array or matlab style + char* fileName = NULL; + if(!mxIsClass(prhs[0], "string")){ + if(!mxIsChar(prhs[0])) mexErrMsgIdAndTxt("tiff:inputError","The first argument must be a string"); + fileName = mxArrayToString(prhs[0]); + } + else{ + mxArray* mString[1]; + mxArray* mCharA[1]; + + // Convert string to char array + mString[0] = mxDuplicateArray(prhs[0]); + mexCallMATLAB(1, mCharA, 1, mString, "char"); + fileName = mxArrayToString(mCharA[0]); + } + + // Handle the tilde character in filenames on Linux/Mac + #ifndef _WIN32 + if(strchr(fileName,'~')) fileName = expandTilde(fileName); + #endif + + uint8_t flipXY = 1; + //uint8_t flipXY = 0; + + + //if(nrhs > 2){ + // flipXY = (uint8_t)*(mxGetPr(prhs[2])); + //} + + + TIFFSetWarningHandler(DummyHandler); + TIFF* tif = TIFFOpen(fileName, "r"); + if(!tif) mexErrMsgIdAndTxt("tiff:inputError","File \"%s\" cannot be opened",fileName); + + uint64_t x = 1,y = 1,z = 1,bits = 1, startSlice = 0; + TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, &x); + TIFFGetField(tif, TIFFTAG_IMAGELENGTH, &y); + + if(nrhs == 1){ + uint16_t s = 0, m = 0, t = 1; + while(TIFFSetDirectory(tif,t)){ + s = t; + t *= 8; + if(s > t){ + t = 65535; + printf("Number of slices > 32768\n"); + break; + } + } + while(s != t){ + m = (s+t+1)/2; + if(TIFFSetDirectory(tif,m)){ + s = m; + } + else{ + if(m > 0) t = m-1; + else t = m; + } + } + z = s+1; + } + else{ + if(mxGetN(prhs[1]) != 2){ + mexErrMsgIdAndTxt("tiff:inputError","Input range is not 2"); + } + else{ + startSlice = (uint64_t)*(mxGetPr(prhs[1]))-1; + z = (uint64_t)*((mxGetPr(prhs[1])+1))-startSlice; + if (!TIFFSetDirectory(tif,startSlice+z-1) || !TIFFSetDirectory(tif,startSlice)){ + mexErrMsgIdAndTxt("tiff:rangeOutOfBound","Range is out of bounds"); + } + } + } + + TIFFGetField(tif, TIFFTAG_BITSPERSAMPLE, &bits); + uint64_t stripSize = 1; + TIFFGetField(tif, TIFFTAG_ROWSPERSTRIP, &stripSize); + TIFFClose(tif); + + uint8_t imageJIm = 0; + if(isImageJIm(fileName)){ + imageJIm = 1; + uint64_t tempZ = imageJImGetZ(fileName); + if(tempZ) z = tempZ; + } + + uint64_t dim[3]; + dim[0] = y; + dim[1] = x; + dim[2] = z; + + + + // Case for ImageJ + if(imageJIm){ + if(bits == 8){ + plhs[0] = mxCreateNumericArray(3,dim,mxUINT8_CLASS, mxREAL); + uint8_t* tiff = (uint8_t*)mxGetPr(plhs[0]); + readTiffParallelImageJ(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else if(bits == 16){ + plhs[0] = mxCreateNumericArray(3,dim,mxUINT16_CLASS, mxREAL); + uint16_t* tiff = (uint16_t*)mxGetPr(plhs[0]); + readTiffParallelImageJ(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else if(bits == 32){ + plhs[0] = mxCreateNumericArray(3,dim,mxSINGLE_CLASS, mxREAL); + float* tiff = (float*)mxGetPr(plhs[0]); + readTiffParallelImageJ(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else if(bits == 64){ + plhs[0] = mxCreateNumericArray(3,dim,mxDOUBLE_CLASS, mxREAL); + double* tiff = (double*)mxGetPr(plhs[0]); + readTiffParallelImageJ(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else{ + mexErrMsgIdAndTxt("tiff:dataTypeError","Data type not suppported"); + } + } + // Case for 2D + else if(z <= 1){ + if(bits == 8){ + plhs[0] = mxCreateNumericArray(3,dim,mxUINT8_CLASS, mxREAL); + uint8_t* tiff = (uint8_t*)mxGetPr(plhs[0]); + readTiffParallel2D(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else if(bits == 16){ + plhs[0] = mxCreateNumericArray(3,dim,mxUINT16_CLASS, mxREAL); + uint16_t* tiff = (uint16_t*)mxGetPr(plhs[0]); + readTiffParallel2D(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else if(bits == 32){ + plhs[0] = mxCreateNumericArray(3,dim,mxSINGLE_CLASS, mxREAL); + float* tiff = (float*)mxGetPr(plhs[0]); + readTiffParallel2D(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else if(bits == 64){ + plhs[0] = mxCreateNumericArray(3,dim,mxDOUBLE_CLASS, mxREAL); + double* tiff = (double*)mxGetPr(plhs[0]); + readTiffParallel2D(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else{ + mexErrMsgIdAndTxt("tiff:dataTypeError","Data type not suppported"); + } + } + // Case for 3D + else{ + if(bits == 8){ + plhs[0] = mxCreateNumericArray(3,dim,mxUINT8_CLASS, mxREAL); + uint8_t* tiff = (uint8_t*)mxGetPr(plhs[0]); + readTiffParallel(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else if(bits == 16){ + plhs[0] = mxCreateNumericArray(3,dim,mxUINT16_CLASS, mxREAL); + uint16_t* tiff = (uint16_t*)mxGetPr(plhs[0]); + readTiffParallel(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else if(bits == 32){ + plhs[0] = mxCreateNumericArray(3,dim,mxSINGLE_CLASS, mxREAL); + float* tiff = (float*)mxGetPr(plhs[0]); + readTiffParallel(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else if(bits == 64){ + plhs[0] = mxCreateNumericArray(3,dim,mxDOUBLE_CLASS, mxREAL); + double* tiff = (double*)mxGetPr(plhs[0]); + readTiffParallel(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); + } + else{ + mexErrMsgIdAndTxt("tiff:dataTypeError","Data type not suppported"); + } + } +} \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h new file mode 100644 index 000000000..e69de29bb diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c new file mode 100644 index 000000000..34912b3f0 --- /dev/null +++ b/tools/llsm_importer.c @@ -0,0 +1,14 @@ +#include +#include +#include + +#define ENABLE_MPI 1 + +#ifdef ENABLE_MPI +#include "mpi.h" +#endif + +#include "pdc.h" +#include "pdc_client_server_common.h" +#include "pdc_client_connect.h" + From 8dfd43f6b5174d4d1d7eed0b77c945cb6ca50d13 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 4 Apr 2023 13:39:36 -0500 Subject: [PATCH 043/216] update server checkpoint intervals --- src/server/pdc_server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 9b5428a5c..f51d00591 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -61,8 +61,8 @@ #include #endif -#define PDC_CHECKPOINT_INTERVAL 200 -#define PDC_CHECKPOINT_MIN_INTERVAL_SEC 300 +#define PDC_CHECKPOINT_INTERVAL 2000 +#define PDC_CHECKPOINT_MIN_INTERVAL_SEC 3600 // Global debug variable to control debug printfs int is_debug_g = 0; From 752aea828a7e8b44f87bb1cf462977c2810adcd2 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 4 Apr 2023 14:07:13 -0500 Subject: [PATCH 044/216] update gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index f706f8e1b..74675f7ef 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ *.pdf src/install + +.vscode + +build \ No newline at end of file From ef7278d8dc11587c7ab064553a6f6709a53cb27b Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 4 Apr 2023 21:18:36 -0500 Subject: [PATCH 045/216] adding job scripts --- scripts/kv_exact/cori/clean.sh | 8 +++ scripts/kv_exact/cori/gen_scripts.sh | 25 +++++++++ scripts/kv_exact/cori/submit.sh | 78 ++++++++++++++++++++++++++++ scripts/kv_exact/cori/template.sh | 67 ++++++++++++++++++++++++ 4 files changed, 178 insertions(+) create mode 100644 scripts/kv_exact/cori/clean.sh create mode 100644 scripts/kv_exact/cori/gen_scripts.sh create mode 100644 scripts/kv_exact/cori/submit.sh create mode 100644 scripts/kv_exact/cori/template.sh diff --git a/scripts/kv_exact/cori/clean.sh b/scripts/kv_exact/cori/clean.sh new file mode 100644 index 000000000..9df3999a6 --- /dev/null +++ b/scripts/kv_exact/cori/clean.sh @@ -0,0 +1,8 @@ +#!/bin/bash +MAX_NODE=512 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + + rm -rf $i/* + +done diff --git a/scripts/kv_exact/cori/gen_scripts.sh b/scripts/kv_exact/cori/gen_scripts.sh new file mode 100644 index 000000000..54606da2b --- /dev/null +++ b/scripts/kv_exact/cori/gen_scripts.sh @@ -0,0 +1,25 @@ +#!/bin/bash +N_THREAD=NO +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + mkdir -p $i + for (( j = 1; j <= $MAX_ATTR; j*=4 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + JOBNAME=kvtag_bench_${i}_${j}_${k} + TARGET=./$i/$JOBNAME.sh + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/ATTRNUM/${j}/g" $TARGET + sed -i "s/ATTRLEN/${k}/g" $TARGET + if [[ "$i" -gt "16" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi + done + done +done diff --git a/scripts/kv_exact/cori/submit.sh b/scripts/kv_exact/cori/submit.sh new file mode 100644 index 000000000..2ca6badf8 --- /dev/null +++ b/scripts/kv_exact/cori/submit.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# MIN_PROC=4 +# MAX_PROC=128 +MIN_PROC=1 +MAX_PROC=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +curdir=$(pwd) + +first_submit=1 + +for (( i = 1; i <= $MAX_PROC; i*=2 )); do + mkdir -p $i + for (( j = 1; j <= $MAX_ATTR; j*=4 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + JOBNAME=kvtag_bench_${i}_${j}_${k} + TARGET=./$i/JOBNAME.sh + + njob=`squeue -u $USER | grep kvtag_bench | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep kvtag_bench | wc -l` + echo $njob + done + + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi + + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime + done + done +done + + +# for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do + +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# while [ $njob -ge 4 ] +# do +# sleeptime=$[ ( $RANDOM % 1000 ) ] +# sleep $sleeptime +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# done + + +# cd $curdir/$j +# for filename in ./*.sh ; do + +# if [[ $first_submit == 1 ]]; then +# # Submit first job w/o dependency +# echo "Submitting $filename" +# job=`sbatch $filename` +# first_submit=0 +# else +# echo "Submitting $filename after ${job: -8}" +# job=`sbatch -d afterany:${job: -8} $filename` +# fi + +# sleeptime=$[ ( $RANDOM % 5 ) ] +# sleep $sleeptime + +# done +# done diff --git a/scripts/kv_exact/cori/template.sh b/scripts/kv_exact/cori/template.sh new file mode 100644 index 000000000..a1da6fda8 --- /dev/null +++ b/scripts/kv_exact/cori/template.sh @@ -0,0 +1,67 @@ +#!/bin/bash -l + +#REGSBATCH -p regular +#DBGSBATCH -p debug +#SBATCH -N NODENUM +#REGSBATCH -t 4:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH --gres=craynetwork:2 +#SBATCH -L SCRATCH +#SBATCH -C haswell +#SBATCH -J JOBNAME +#SBATCH -A m2621 +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + + +# export PDC_DEBUG=0 + +export PDC_TMPDIR=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/conf + +rm -rf $PDC_TMPDIR/* + +REPEAT=1 + +N_NODE=NODENUM +NCLIENT=31 + +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE +mkdir -p $PDC_TMPDIR + +let TOTALPROC=$NCLIENT*$N_NODE + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/share/test/bin +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$EXECPATH/kvtag_scale_add_get +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* + +MAX_OBJ_COUNT=$((1024*1024*1024)) +OBJ_INCR=$((MAX_OBJ_COUNT/1024)) +ATTR_COUNT=ATTRNUM +ATTR_LENGTH=ATTRLEN +QUERY_COUNT=$((1024*1024)) + +date + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +srun -N $N_NODE -n $N_NODE -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +srun -N 1 -n 1 -c 2 --mem=25600 --gres=craynetwork:1 --cpu_bind=cores --overlap $CLOSE + +date From d34ce8b30b2562fc5c0336bf4c249c7ce7b5b6f9 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 13:15:10 -0500 Subject: [PATCH 046/216] adding one debugging msg --- scripts/kv_exact/cori/clean.sh | 2 +- src/api/pdc_client_connect.c | 3 +++ src/tests/kvtag_add_get_benchmark.c | 2 +- src/utils/include/pdc_private.h | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/kv_exact/cori/clean.sh b/scripts/kv_exact/cori/clean.sh index 9df3999a6..d6476962e 100644 --- a/scripts/kv_exact/cori/clean.sh +++ b/scripts/kv_exact/cori/clean.sh @@ -5,4 +5,4 @@ for (( i = 1; i <= $MAX_NODE; i*=2 )); do rm -rf $i/* -done +done \ No newline at end of file diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index 0dde1b5a6..ed59e565f 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -6960,6 +6960,9 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) in.hash_value = PDC_get_hash_by_name(cont_prop->cont_info_pub->name); } + //TODO: delete this line after debugging. + printf("PDC_add_kvtag::in.obj_id = %llu \n ", in.obj_id); + server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); // Debug statistics for counting number of messages sent to each server. diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 5586f4b6c..fa8b1eca1 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -221,7 +221,7 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value for (j = 0; j < n_attr; j++) { sprintf(tag_name, "tag%llu.%llu", v, j); if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], tag_value_len + 1) < 0) - printf("fail to add a kvtag to o%llu\n", i + my_obj_s); + printf("fail to add a kvtag to o%llu\n", v); } } } diff --git a/src/utils/include/pdc_private.h b/src/utils/include/pdc_private.h index 5f5a2712f..b6ca3bcfd 100644 --- a/src/utils/include/pdc_private.h +++ b/src/utils/include/pdc_private.h @@ -28,6 +28,7 @@ #include "pdc_config.h" #include "pdc_public.h" #include +// #include /* gettimeofday() */ /****************************/ /* Library Private Typedefs */ From d78533e3cc12898a23a2769c96990d5e2de20e92 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 14:56:00 -0500 Subject: [PATCH 047/216] update container creation to collective mode for debugging purpose --- src/tests/kvtag_add_get_benchmark.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index fa8b1eca1..f0e7e7110 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -156,7 +156,7 @@ init_test(int my_rank, int proc_num, uint64_t n_obj_incr, uint64_t n_query, uint printf("Fail to create container property @ line %d!\n", __LINE__); // create a container - *cont = PDCcont_create("c1", *cont_prop); + *cont = PDCcont_create_col("c1", *cont_prop); if (*cont <= 0) printf("Fail to create container @ line %d!\n", __LINE__); From 334efc4748145da09e13a7549e35c286f2af1ef5 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 15:04:24 -0500 Subject: [PATCH 048/216] update container creation to collective mode for debugging purpose --- src/api/pdc_client_connect.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index ed59e565f..897379e77 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -6961,7 +6961,7 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) } //TODO: delete this line after debugging. - printf("PDC_add_kvtag::in.obj_id = %llu \n ", in.obj_id); + printf("==CLIENT[%d]: PDC_add_kvtag::in.obj_id = %llu \n ", pdc_client_mpi_rank_g, in.obj_id); server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); From c704bc6b4f1581ef8353d69f009edf393378e3b7 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 15:37:20 -0500 Subject: [PATCH 049/216] update container creation to collective mode for debugging purpose --- src/server/pdc_server_metadata.c | 2 ++ src/tests/kvtag_add_get_benchmark.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/pdc_server_metadata.c b/src/server/pdc_server_metadata.c index 7253a8f1c..e2411fc2d 100644 --- a/src/server/pdc_server_metadata.c +++ b/src/server/pdc_server_metadata.c @@ -2555,6 +2555,8 @@ PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out) hash_key = in->hash_value; obj_id = in->obj_id; + printf("==SERVER[%d]: PDC_add_kvtag::in.obj_id = %llu \n ", pdc_server_rank_g, in.obj_id); + #ifdef ENABLE_MULTITHREAD // Obtain lock for hash table unlocked = 0; diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index f0e7e7110..fa8b1eca1 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -156,7 +156,7 @@ init_test(int my_rank, int proc_num, uint64_t n_obj_incr, uint64_t n_query, uint printf("Fail to create container property @ line %d!\n", __LINE__); // create a container - *cont = PDCcont_create_col("c1", *cont_prop); + *cont = PDCcont_create("c1", *cont_prop); if (*cont <= 0) printf("Fail to create container @ line %d!\n", __LINE__); From 0139db71ad2e679c6717c9844663531791e1323f Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 15:39:35 -0500 Subject: [PATCH 050/216] update container creation to collective mode for debugging purpose --- src/server/pdc_server_metadata.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/pdc_server_metadata.c b/src/server/pdc_server_metadata.c index e2411fc2d..79cd22dfd 100644 --- a/src/server/pdc_server_metadata.c +++ b/src/server/pdc_server_metadata.c @@ -2555,7 +2555,7 @@ PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out) hash_key = in->hash_value; obj_id = in->obj_id; - printf("==SERVER[%d]: PDC_add_kvtag::in.obj_id = %llu \n ", pdc_server_rank_g, in.obj_id); + printf("==SERVER[%d]: PDC_add_kvtag::in.obj_id = %llu \n ", pdc_server_rank_g, obj_id); #ifdef ENABLE_MULTITHREAD // Obtain lock for hash table From d0b47a62a0a03a0dda16d6a6b0f7c20e580f94b7 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 21:33:13 -0500 Subject: [PATCH 051/216] update output for uint64_t --- scripts/kv_exact/cori/clean.sh | 8 --- scripts/kv_exact/cori/gen_scripts.sh | 25 --------- scripts/kv_exact/cori/submit.sh | 78 ---------------------------- scripts/kv_exact/cori/template.sh | 67 ------------------------ src/tests/kvtag_add_get_benchmark.c | 53 ++++++++++--------- 5 files changed, 29 insertions(+), 202 deletions(-) delete mode 100644 scripts/kv_exact/cori/clean.sh delete mode 100644 scripts/kv_exact/cori/gen_scripts.sh delete mode 100644 scripts/kv_exact/cori/submit.sh delete mode 100644 scripts/kv_exact/cori/template.sh diff --git a/scripts/kv_exact/cori/clean.sh b/scripts/kv_exact/cori/clean.sh deleted file mode 100644 index d6476962e..000000000 --- a/scripts/kv_exact/cori/clean.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -MAX_NODE=512 - -for (( i = 1; i <= $MAX_NODE; i*=2 )); do - - rm -rf $i/* - -done \ No newline at end of file diff --git a/scripts/kv_exact/cori/gen_scripts.sh b/scripts/kv_exact/cori/gen_scripts.sh deleted file mode 100644 index 54606da2b..000000000 --- a/scripts/kv_exact/cori/gen_scripts.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -N_THREAD=NO -MAX_NODE=512 -MAX_ATTR=1024 -MAX_ATTRLEN=1000 - -for (( i = 1; i <= $MAX_NODE; i*=2 )); do - mkdir -p $i - for (( j = 1; j <= $MAX_ATTR; j*=4 )); do - for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do - JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/$JOBNAME.sh - cp template.sh $TARGET - sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET - sed -i "s/NODENUM/${i}/g" $TARGET - sed -i "s/ATTRNUM/${j}/g" $TARGET - sed -i "s/ATTRLEN/${k}/g" $TARGET - if [[ "$i" -gt "16" ]]; then - sed -i "s/REG//g" $TARGET - else - sed -i "s/DBG//g" $TARGET - fi - done - done -done diff --git a/scripts/kv_exact/cori/submit.sh b/scripts/kv_exact/cori/submit.sh deleted file mode 100644 index 2ca6badf8..000000000 --- a/scripts/kv_exact/cori/submit.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# MIN_PROC=4 -# MAX_PROC=128 -MIN_PROC=1 -MAX_PROC=512 -MAX_ATTR=1024 -MAX_ATTRLEN=1000 - -curdir=$(pwd) - -first_submit=1 - -for (( i = 1; i <= $MAX_PROC; i*=2 )); do - mkdir -p $i - for (( j = 1; j <= $MAX_ATTR; j*=4 )); do - for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do - JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/JOBNAME.sh - - njob=`squeue -u $USER | grep kvtag_bench | wc -l` - echo $njob - while [ $njob -ge 4 ] - do - sleeptime=$[ ( $RANDOM % 1000 ) ] - sleep $sleeptime - njob=`squeue -u $USER | grep kvtag_bench | wc -l` - echo $njob - done - - if [[ $first_submit == 1 ]]; then - # Submit first job w/o dependency - echo "Submitting $TARGET" - job=`sbatch $TARGET` - first_submit=0 - else - echo "Submitting $TARGET after ${job: -8}" - job=`sbatch -d afterany:${job: -8} $TARGET` - fi - - sleeptime=$[ ( $RANDOM % 5 ) ] - sleep $sleeptime - done - done -done - - -# for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do - -# njob=`squeue -u $USER | grep vpic | wc -l` -# echo $njob -# while [ $njob -ge 4 ] -# do -# sleeptime=$[ ( $RANDOM % 1000 ) ] -# sleep $sleeptime -# njob=`squeue -u $USER | grep vpic | wc -l` -# echo $njob -# done - - -# cd $curdir/$j -# for filename in ./*.sh ; do - -# if [[ $first_submit == 1 ]]; then -# # Submit first job w/o dependency -# echo "Submitting $filename" -# job=`sbatch $filename` -# first_submit=0 -# else -# echo "Submitting $filename after ${job: -8}" -# job=`sbatch -d afterany:${job: -8} $filename` -# fi - -# sleeptime=$[ ( $RANDOM % 5 ) ] -# sleep $sleeptime - -# done -# done diff --git a/scripts/kv_exact/cori/template.sh b/scripts/kv_exact/cori/template.sh deleted file mode 100644 index a1da6fda8..000000000 --- a/scripts/kv_exact/cori/template.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -l - -#REGSBATCH -p regular -#DBGSBATCH -p debug -#SBATCH -N NODENUM -#REGSBATCH -t 4:00:00 -#DBGSBATCH -t 0:30:00 -#SBATCH --gres=craynetwork:2 -#SBATCH -L SCRATCH -#SBATCH -C haswell -#SBATCH -J JOBNAME -#SBATCH -A m2621 -#SBATCH -o o%j.JOBNAME.out -#SBATCH -e o%j.JOBNAME.out - - -# export PDC_DEBUG=0 - -export PDC_TMPDIR=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/conf - -rm -rf $PDC_TMPDIR/* - -REPEAT=1 - -N_NODE=NODENUM -NCLIENT=31 - -export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE -mkdir -p $PDC_TMPDIR - -let TOTALPROC=$NCLIENT*$N_NODE - -EXECPATH=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/share/test/bin -SERVER=$EXECPATH/pdc_server.exe -CLIENT=$EXECPATH/kvtag_scale_add_get -CLOSE=$EXECPATH/close_server - -chmod +x $EXECPATH/* - -MAX_OBJ_COUNT=$((1024*1024*1024)) -OBJ_INCR=$((MAX_OBJ_COUNT/1024)) -ATTR_COUNT=ATTRNUM -ATTR_LENGTH=ATTRLEN -QUERY_COUNT=$((1024*1024)) - -date - -echo "" -echo "=============" -echo "$i Init server" -echo "=============" -srun -N $N_NODE -n $N_NODE -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap $SERVER & -sleep 5 - - -echo "============================================" -echo "KVTAGS with $N_NODE nodes" -echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE - -echo "" -echo "=================" -echo "$i Closing server" -echo "=================" -srun -N 1 -n 1 -c 2 --mem=25600 --gres=craynetwork:1 --cpu_bind=cores --overlap $CLOSE - -date diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index fa8b1eca1..bce854644 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -61,7 +61,7 @@ atoui64(char *arg) * @return 0 if the function executes successfully, non-zero if an error occurs. */ int -assign_work_to_rank(int rank, int size, uint64_t nwork, uint64_t *my_count, uint64_t *my_start) +assign_work_to_rank(uint64_t rank, uint64_t size, uint64_t nwork, uint64_t *my_count, uint64_t *my_start) { if (rank > size || my_count == NULL || my_start == NULL) { printf("assign_work_to_rank(): Error with input!\n"); @@ -169,8 +169,8 @@ init_test(int my_rank, int proc_num, uint64_t n_obj_incr, uint64_t n_query, uint printf("create obj_ids array\n"); // Create a number of objects, add at least one tag to that object - assign_work_to_rank(my_rank, proc_num, n_obj_incr, my_obj, my_obj_s); - assign_work_to_rank(my_rank, proc_num, n_query, my_query, my_query_s); + assign_work_to_rank((uint64_t)my_rank, (uint64_t)proc_num, n_obj_incr, my_obj, my_obj_s); + assign_work_to_rank((uint64_t)my_rank, (uint64_t)proc_num, n_query, my_query, my_query_s); return (pdcid_t *)calloc(*my_obj, sizeof(pdcid_t)); } @@ -219,9 +219,9 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value for (i = 0; i < my_obj; i++) { v = i + my_obj_s; for (j = 0; j < n_attr; j++) { - sprintf(tag_name, "tag%llu.%llu", v, j); + sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", v, i); if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], tag_value_len + 1) < 0) - printf("fail to add a kvtag to o%llu\n", v); + printf("fail to add a kvtag to o%" PRIu64 "\n", v); } } } @@ -242,9 +242,9 @@ get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, int n_attr, void **tag_valu char tag_name[128]; for (i = 0; i < n_attr; i++) { - sprintf(tag_name, "tag%llu.%llu", obj_name_v, i); + sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", obj_name_v, i); if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], (void *)&value_size[i]) < 0) - printf("fail to get a kvtag from o%llu\n", v); + printf("fail to get a kvtag from o%" PRIu64 "\n", v); } } @@ -286,8 +286,8 @@ check_and_release_query_result(uint64_t n_query, uint64_t my_obj, uint64_t my_ob for (j = 0; j < n_attr; j++) { char *query_rst = (char *)values[j + i * n_attr]; if (strcmp(query_rst, tag_values[j]) != 0) { - printf("Error with retrieved tag from o%llu. Expected %s, Found %s \n", v, tag_values[j], - query_rst); + printf("Error with retrieved tag from o%" PRIu64 ". Expected %s, Found %s \n", v, + tag_values[j], query_rst); } free(values[j + i * n_attr]); } @@ -297,7 +297,7 @@ check_and_release_query_result(uint64_t n_query, uint64_t my_obj, uint64_t my_ob for (i = 0; i < my_obj; i++) { v = i + my_obj_s; if (PDCobj_close(obj_ids[i]) < 0) - printf("fail to close object o%llu\n", v); + printf("fail to close object o%" PRIu64 "\n", v); } } @@ -395,10 +395,12 @@ main(int argc, char *argv[]) total_object_count += n_obj_incr; #endif if (my_rank == 0) { - printf("Iteration %4llu : Objects: %6llu , Time: %.4f sec. Object throughput in this iteration: " + printf("Iteration %" PRIu64 " : Objects: %" PRIu64 + " , Time: %.4f sec. Object throughput in this iteration: " "%.4f .\n", k, n_obj_incr, step_elapse, ((double)n_obj_incr) / step_elapse); - printf("Overall %4llu : Objects: %6llu , Time: %.4f sec. Overall object throughput: " + printf("Overall %" PRIu64 " : Objects: %" PRIu64 + " , Time: %.4f sec. Overall object throughput: " "%.4f .\n", k, total_object_count, total_object_time, ((double)total_object_count) / total_object_time); @@ -418,12 +420,12 @@ main(int argc, char *argv[]) total_tag_count += n_obj_incr * n_attr; #endif if (my_rank == 0) { - printf( - "Iteration %4llu : Tags: %6llu , Time: %.4f sec. Tag throughput in this iteration: %.4f .\n", - k, n_obj_incr * n_attr, step_elapse, (double)(n_obj_incr * n_attr) / step_elapse); - printf( - "Overall %4llu : Tags: %6llu , Time: %.4f sec. Overall tag throughput: %.4f .\n", - k, total_tag_count, total_tag_time, ((double)total_tag_count) / total_tag_time); + printf("Iteration %" PRIu64 " : Tags: %" PRIu64 + " , Time: %.4f sec. Tag throughput in this iteration: %.4f .\n", + k, n_obj_incr * n_attr, step_elapse, (double)(n_obj_incr * n_attr) / step_elapse); + printf("Overall %" PRIu64 " : Tags: %" PRIu64 + " , Time: %.4f sec. Overall tag throughput: %.4f .\n", + k, total_tag_count, total_tag_time, ((double)total_tag_count) / total_tag_time); } query_rst_cache = (void **)malloc(my_query * n_attr * sizeof(void *)); @@ -442,10 +444,12 @@ main(int argc, char *argv[]) total_query_count += n_query * n_attr; #endif if (my_rank == 0) { - printf("Iteration %4llu : Queries: %6llu , Time: %.4f sec. Query throughput in this iteration: " + printf("Iteration %" PRIu64 " : Queries: %" PRIu64 + " , Time: %.4f sec. Query throughput in this iteration: " "%.4f .\n", k, n_query * n_attr, step_elapse, (double)(n_query * n_attr) / step_elapse); - printf("Overall %4llu : Queries: %6llu , Time: %.4f sec. Overall query throughput: " + printf("Overall %" PRIu64 " : Queries: %" PRIu64 + " , Time: %.4f sec. Overall query throughput: " "%.4f .\n", k, total_query_count, total_query_time, ((double)total_query_count) / total_query_time); } @@ -461,10 +465,11 @@ main(int argc, char *argv[]) if (my_rank == 0) { printf("Final Report: \n"); - printf("[Final Report 1] Servers: %llu , Clients: %llu , C/S ratio: %.4f \n", n_servers, n_clients, - (double)n_clients / (double)n_servers); - printf("[Final Report 2] Iterations: %llu , Objects: %llu , Tags/Object: %llu , Queries/Iteration: " - "%llu , \n", + printf("[Final Report 1] Servers: %" PRIu64 " , Clients: %" PRIu64 " , C/S ratio: %.4f \n", n_servers, + n_clients, (double)n_clients / (double)n_servers); + printf("[Final Report 2] Iterations: %" PRIu64 " , Objects: %" PRIu64 " , Tags/Object: %" PRIu64 + " , Queries/Iteration: " + "%" PRIu64 " , \n", k, curr_total_obj, n_attr, n_query); printf("[Final Report 3] Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", (double)curr_total_obj / total_object_time, (double)(curr_total_obj * n_attr) / total_tag_time, From 70ffe9daa55cbd1d1e46ce4df32b7030193461fd Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 21:33:34 -0500 Subject: [PATCH 052/216] add scripts --- scripts/kvtag_add_get_benchmark/cori/clean.sh | 8 ++ .../cori/gen_scripts.sh | 25 ++++++ .../kvtag_add_get_benchmark/cori/submit.sh | 78 +++++++++++++++++++ .../kvtag_add_get_benchmark/cori/template.sh | 68 ++++++++++++++++ scripts/kvtag_add_get_scale/cori/clean.sh | 8 ++ .../kvtag_add_get_scale/cori/gen_scripts.sh | 25 ++++++ scripts/kvtag_add_get_scale/cori/submit.sh | 78 +++++++++++++++++++ scripts/kvtag_add_get_scale/cori/template.sh | 68 ++++++++++++++++ .../kvtag_add_get_scale/perlmutter/clean.sh | 8 ++ .../perlmutter/gen_scripts.sh | 25 ++++++ .../kvtag_add_get_scale/perlmutter/submit.sh | 78 +++++++++++++++++++ .../perlmutter/template.sh | 69 ++++++++++++++++ 12 files changed, 538 insertions(+) create mode 100644 scripts/kvtag_add_get_benchmark/cori/clean.sh create mode 100644 scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh create mode 100644 scripts/kvtag_add_get_benchmark/cori/submit.sh create mode 100644 scripts/kvtag_add_get_benchmark/cori/template.sh create mode 100644 scripts/kvtag_add_get_scale/cori/clean.sh create mode 100644 scripts/kvtag_add_get_scale/cori/gen_scripts.sh create mode 100644 scripts/kvtag_add_get_scale/cori/submit.sh create mode 100644 scripts/kvtag_add_get_scale/cori/template.sh create mode 100644 scripts/kvtag_add_get_scale/perlmutter/clean.sh create mode 100644 scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh create mode 100644 scripts/kvtag_add_get_scale/perlmutter/submit.sh create mode 100644 scripts/kvtag_add_get_scale/perlmutter/template.sh diff --git a/scripts/kvtag_add_get_benchmark/cori/clean.sh b/scripts/kvtag_add_get_benchmark/cori/clean.sh new file mode 100644 index 000000000..d6476962e --- /dev/null +++ b/scripts/kvtag_add_get_benchmark/cori/clean.sh @@ -0,0 +1,8 @@ +#!/bin/bash +MAX_NODE=512 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + + rm -rf $i/* + +done \ No newline at end of file diff --git a/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh b/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh new file mode 100644 index 000000000..54606da2b --- /dev/null +++ b/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh @@ -0,0 +1,25 @@ +#!/bin/bash +N_THREAD=NO +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + mkdir -p $i + for (( j = 1; j <= $MAX_ATTR; j*=4 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + JOBNAME=kvtag_bench_${i}_${j}_${k} + TARGET=./$i/$JOBNAME.sh + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/ATTRNUM/${j}/g" $TARGET + sed -i "s/ATTRLEN/${k}/g" $TARGET + if [[ "$i" -gt "16" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi + done + done +done diff --git a/scripts/kvtag_add_get_benchmark/cori/submit.sh b/scripts/kvtag_add_get_benchmark/cori/submit.sh new file mode 100644 index 000000000..2ca6badf8 --- /dev/null +++ b/scripts/kvtag_add_get_benchmark/cori/submit.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# MIN_PROC=4 +# MAX_PROC=128 +MIN_PROC=1 +MAX_PROC=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +curdir=$(pwd) + +first_submit=1 + +for (( i = 1; i <= $MAX_PROC; i*=2 )); do + mkdir -p $i + for (( j = 1; j <= $MAX_ATTR; j*=4 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + JOBNAME=kvtag_bench_${i}_${j}_${k} + TARGET=./$i/JOBNAME.sh + + njob=`squeue -u $USER | grep kvtag_bench | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep kvtag_bench | wc -l` + echo $njob + done + + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi + + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime + done + done +done + + +# for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do + +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# while [ $njob -ge 4 ] +# do +# sleeptime=$[ ( $RANDOM % 1000 ) ] +# sleep $sleeptime +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# done + + +# cd $curdir/$j +# for filename in ./*.sh ; do + +# if [[ $first_submit == 1 ]]; then +# # Submit first job w/o dependency +# echo "Submitting $filename" +# job=`sbatch $filename` +# first_submit=0 +# else +# echo "Submitting $filename after ${job: -8}" +# job=`sbatch -d afterany:${job: -8} $filename` +# fi + +# sleeptime=$[ ( $RANDOM % 5 ) ] +# sleep $sleeptime + +# done +# done diff --git a/scripts/kvtag_add_get_benchmark/cori/template.sh b/scripts/kvtag_add_get_benchmark/cori/template.sh new file mode 100644 index 000000000..73eb670c5 --- /dev/null +++ b/scripts/kvtag_add_get_benchmark/cori/template.sh @@ -0,0 +1,68 @@ +#!/bin/bash -l + +#REGSBATCH -p regular +#DBGSBATCH -p debug +#SBATCH -N NODENUM +#REGSBATCH -t 4:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH --gres=craynetwork:2 +#SBATCH -L SCRATCH +#SBATCH -C haswell +#SBATCH -J JOBNAME +#SBATCH -A m2621 +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + + +# export PDC_DEBUG=0 + +export PDC_TMPDIR=/global/cscratch1/sd/wzhang5/data/pdc/conf + +rm -rf $PDC_TMPDIR/* + +REPEAT=1 + +N_NODE=NODENUM +NCLIENT=31 + +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE +mkdir -p $PDC_TMPDIR + +let TOTALPROC=$NCLIENT*$N_NODE + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/share/test/bin +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$EXECPATH/kvtag_add_get_benchmark +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* + +MAX_OBJ_COUNT=$((1024*1024)) +OBJ_INCR=$((MAX_OBJ_COUNT/1024)) +ATTR_COUNT=ATTRNUM +ATTR_LENGTH=ATTRLEN +QUERY_COUNT=$((1024)) + +date + + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +srun -N $N_NODE -n $N_NODE -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap stdbuf -i0 -o0 -e0 $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap stdbuf -i0 -o0 -e0 $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +srun -N 1 -n 1 -c 2 --mem=25600 --gres=craynetwork:1 --cpu_bind=cores --overlap stdbuf -i0 -o0 -e0 $CLOSE + +date diff --git a/scripts/kvtag_add_get_scale/cori/clean.sh b/scripts/kvtag_add_get_scale/cori/clean.sh new file mode 100644 index 000000000..d6476962e --- /dev/null +++ b/scripts/kvtag_add_get_scale/cori/clean.sh @@ -0,0 +1,8 @@ +#!/bin/bash +MAX_NODE=512 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + + rm -rf $i/* + +done \ No newline at end of file diff --git a/scripts/kvtag_add_get_scale/cori/gen_scripts.sh b/scripts/kvtag_add_get_scale/cori/gen_scripts.sh new file mode 100644 index 000000000..54606da2b --- /dev/null +++ b/scripts/kvtag_add_get_scale/cori/gen_scripts.sh @@ -0,0 +1,25 @@ +#!/bin/bash +N_THREAD=NO +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + mkdir -p $i + for (( j = 1; j <= $MAX_ATTR; j*=4 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + JOBNAME=kvtag_bench_${i}_${j}_${k} + TARGET=./$i/$JOBNAME.sh + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/ATTRNUM/${j}/g" $TARGET + sed -i "s/ATTRLEN/${k}/g" $TARGET + if [[ "$i" -gt "16" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi + done + done +done diff --git a/scripts/kvtag_add_get_scale/cori/submit.sh b/scripts/kvtag_add_get_scale/cori/submit.sh new file mode 100644 index 000000000..2ca6badf8 --- /dev/null +++ b/scripts/kvtag_add_get_scale/cori/submit.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# MIN_PROC=4 +# MAX_PROC=128 +MIN_PROC=1 +MAX_PROC=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +curdir=$(pwd) + +first_submit=1 + +for (( i = 1; i <= $MAX_PROC; i*=2 )); do + mkdir -p $i + for (( j = 1; j <= $MAX_ATTR; j*=4 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + JOBNAME=kvtag_bench_${i}_${j}_${k} + TARGET=./$i/JOBNAME.sh + + njob=`squeue -u $USER | grep kvtag_bench | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep kvtag_bench | wc -l` + echo $njob + done + + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi + + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime + done + done +done + + +# for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do + +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# while [ $njob -ge 4 ] +# do +# sleeptime=$[ ( $RANDOM % 1000 ) ] +# sleep $sleeptime +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# done + + +# cd $curdir/$j +# for filename in ./*.sh ; do + +# if [[ $first_submit == 1 ]]; then +# # Submit first job w/o dependency +# echo "Submitting $filename" +# job=`sbatch $filename` +# first_submit=0 +# else +# echo "Submitting $filename after ${job: -8}" +# job=`sbatch -d afterany:${job: -8} $filename` +# fi + +# sleeptime=$[ ( $RANDOM % 5 ) ] +# sleep $sleeptime + +# done +# done diff --git a/scripts/kvtag_add_get_scale/cori/template.sh b/scripts/kvtag_add_get_scale/cori/template.sh new file mode 100644 index 000000000..73eb670c5 --- /dev/null +++ b/scripts/kvtag_add_get_scale/cori/template.sh @@ -0,0 +1,68 @@ +#!/bin/bash -l + +#REGSBATCH -p regular +#DBGSBATCH -p debug +#SBATCH -N NODENUM +#REGSBATCH -t 4:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH --gres=craynetwork:2 +#SBATCH -L SCRATCH +#SBATCH -C haswell +#SBATCH -J JOBNAME +#SBATCH -A m2621 +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + + +# export PDC_DEBUG=0 + +export PDC_TMPDIR=/global/cscratch1/sd/wzhang5/data/pdc/conf + +rm -rf $PDC_TMPDIR/* + +REPEAT=1 + +N_NODE=NODENUM +NCLIENT=31 + +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE +mkdir -p $PDC_TMPDIR + +let TOTALPROC=$NCLIENT*$N_NODE + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/share/test/bin +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$EXECPATH/kvtag_add_get_benchmark +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* + +MAX_OBJ_COUNT=$((1024*1024)) +OBJ_INCR=$((MAX_OBJ_COUNT/1024)) +ATTR_COUNT=ATTRNUM +ATTR_LENGTH=ATTRLEN +QUERY_COUNT=$((1024)) + +date + + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +srun -N $N_NODE -n $N_NODE -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap stdbuf -i0 -o0 -e0 $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap stdbuf -i0 -o0 -e0 $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +srun -N 1 -n 1 -c 2 --mem=25600 --gres=craynetwork:1 --cpu_bind=cores --overlap stdbuf -i0 -o0 -e0 $CLOSE + +date diff --git a/scripts/kvtag_add_get_scale/perlmutter/clean.sh b/scripts/kvtag_add_get_scale/perlmutter/clean.sh new file mode 100644 index 000000000..d6476962e --- /dev/null +++ b/scripts/kvtag_add_get_scale/perlmutter/clean.sh @@ -0,0 +1,8 @@ +#!/bin/bash +MAX_NODE=512 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + + rm -rf $i/* + +done \ No newline at end of file diff --git a/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh b/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh new file mode 100644 index 000000000..54606da2b --- /dev/null +++ b/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh @@ -0,0 +1,25 @@ +#!/bin/bash +N_THREAD=NO +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + mkdir -p $i + for (( j = 1; j <= $MAX_ATTR; j*=4 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + JOBNAME=kvtag_bench_${i}_${j}_${k} + TARGET=./$i/$JOBNAME.sh + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/ATTRNUM/${j}/g" $TARGET + sed -i "s/ATTRLEN/${k}/g" $TARGET + if [[ "$i" -gt "16" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi + done + done +done diff --git a/scripts/kvtag_add_get_scale/perlmutter/submit.sh b/scripts/kvtag_add_get_scale/perlmutter/submit.sh new file mode 100644 index 000000000..2ca6badf8 --- /dev/null +++ b/scripts/kvtag_add_get_scale/perlmutter/submit.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# MIN_PROC=4 +# MAX_PROC=128 +MIN_PROC=1 +MAX_PROC=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +curdir=$(pwd) + +first_submit=1 + +for (( i = 1; i <= $MAX_PROC; i*=2 )); do + mkdir -p $i + for (( j = 1; j <= $MAX_ATTR; j*=4 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + JOBNAME=kvtag_bench_${i}_${j}_${k} + TARGET=./$i/JOBNAME.sh + + njob=`squeue -u $USER | grep kvtag_bench | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep kvtag_bench | wc -l` + echo $njob + done + + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi + + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime + done + done +done + + +# for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do + +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# while [ $njob -ge 4 ] +# do +# sleeptime=$[ ( $RANDOM % 1000 ) ] +# sleep $sleeptime +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# done + + +# cd $curdir/$j +# for filename in ./*.sh ; do + +# if [[ $first_submit == 1 ]]; then +# # Submit first job w/o dependency +# echo "Submitting $filename" +# job=`sbatch $filename` +# first_submit=0 +# else +# echo "Submitting $filename after ${job: -8}" +# job=`sbatch -d afterany:${job: -8} $filename` +# fi + +# sleeptime=$[ ( $RANDOM % 5 ) ] +# sleep $sleeptime + +# done +# done diff --git a/scripts/kvtag_add_get_scale/perlmutter/template.sh b/scripts/kvtag_add_get_scale/perlmutter/template.sh new file mode 100644 index 000000000..56fa7cd02 --- /dev/null +++ b/scripts/kvtag_add_get_scale/perlmutter/template.sh @@ -0,0 +1,69 @@ +#!/bin/bash -l + +#REGSBATCH -p regular +#DBGSBATCH -p debug +#SBATCH -N NODENUM +#REGSBATCH -t 4:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH --gres=craynetwork:2 +#SBATCH -L SCRATCH +#SBATCH -C haswell +#SBATCH -J JOBNAME +#SBATCH -A m2621 +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + + +# export PDC_DEBUG=0 + +export PDC_TMPDIR=/global/cscratch1/sd/wzhang5/data/pdc/conf + +rm -rf $PDC_TMPDIR/* + +REPEAT=1 + +N_NODE=NODENUM +NCLIENT=31 + +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE +mkdir -p $PDC_TMPDIR + +let TOTALPROC=$NCLIENT*$N_NODE + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/share/test/bin +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$EXECPATH/kvtag_add_get_benchmark +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* + +NUM_OBJ= +MAX_OBJ_COUNT=$((1024*1024)) +OBJ_INCR=$((MAX_OBJ_COUNT/1024)) +ATTR_COUNT=ATTRNUM +ATTR_LENGTH=ATTRLEN +QUERY_COUNT=$((1024)) + +date + + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +srun -N $N_NODE -n $N_NODE -c 2 --mem=128000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +srun -N $N_NODE -n $TOTALPROC -c 2 --mem=256000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLOSE + +date From 90736ab646022f7d934c17f6678b0cd6924fd1d9 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 21:42:34 -0500 Subject: [PATCH 053/216] update output for uint64_t --- src/tests/kvtag_add_get_benchmark.c | 30 +++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index bce854644..9f39293e6 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -192,7 +192,7 @@ create_object(uint64_t my_obj, uint64_t my_obj_s, pdcid_t cont, pdcid_t obj_prop for (i = 0; i < my_obj; i++) { v = i + my_obj_s; - sprintf(obj_name, "obj%llu", v); + sprintf(obj_name, "obj%" PRIu64 "", v); obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); if (obj_ids[i] <= 0) printf("Fail to create object @ line %d!\n", __LINE__); @@ -236,15 +236,15 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value * @param value_size An array to store the size of each tag value. */ void -get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, int n_attr, void **tag_values, uint64_t *value_size) +get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, uint64_t n_attr, void **tag_values, uint64_t *value_size) { - uint64_t i, v; + uint64_t i; char tag_name[128]; for (i = 0; i < n_attr; i++) { sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", obj_name_v, i); if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], (void *)&value_size[i]) < 0) - printf("fail to get a kvtag from o%" PRIu64 "\n", v); + printf("fail to get a kvtag from o%" PRIu64 "\n", obj_name_v); } } @@ -264,7 +264,7 @@ get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, int n_attr, void **tag_valu * The caller is responsible for allocating memory for the array. */ void -send_queries(uint64_t my_obj_s, int n_query, uint64_t n_attr, pdcid_t *obj_ids, void **tag_values, +send_queries(uint64_t my_obj_s, uint64_t n_query, uint64_t n_attr, pdcid_t *obj_ids, void **tag_values, uint64_t *value_size) { uint64_t i, v; @@ -325,21 +325,17 @@ main(int argc, char *argv[]) { pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t * obj_ids; - uint64_t n_obj, n_obj_incr, my_obj, my_obj_s, curr_total_obj = 0; + uint64_t n_obj, n_obj_incr, my_obj, my_obj_s; uint64_t n_attr, n_attr_len, n_query, my_query, my_query_s; uint64_t n_servers, n_clients; - uint64_t i, j, k, v; - int proc_num, my_rank, attr_value; - char obj_name[128]; - char tag_name[128]; + uint64_t i, k; + int proc_num, my_rank; double stime = 0.0, step_elapse = 0.0; double total_object_time = 0.0, total_tag_time = 0.0, total_query_time = 0.0; uint64_t total_object_count = 0, total_tag_count = 0, total_query_count = 0; - int * value_to_add; void ** query_rst_cache; uint64_t * value_size; - obj_handle * oh; - struct pdc_obj_info *info; + #ifdef ENABLE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); @@ -365,7 +361,7 @@ main(int argc, char *argv[]) } if (my_rank == 0) - printf("Create %llu obj, %llu tags, query %llu\n", n_obj, n_obj, n_obj); + printf("Create %" PRIu64 " obj, %" PRIu64 " tags, query %" PRIu64 "\n", n_obj, n_attr, n_query); // making necessary preparation for the test. @@ -386,14 +382,14 @@ main(int argc, char *argv[]) // n_obj_incr. create_object(my_obj, my_obj_s, cont, obj_prop, obj_ids); // therefore, after 'create_objects' function, we should add 'curr_total_obj' by 'n_obj_incr'. - curr_total_obj += n_obj_incr; #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); step_elapse = MPI_Wtime() - stime; total_object_time += step_elapse; - total_object_count += n_obj_incr; #endif + total_object_count += n_obj_incr; + if (my_rank == 0) { printf("Iteration %" PRIu64 " : Objects: %" PRIu64 " , Time: %.4f sec. Object throughput in this iteration: " @@ -461,7 +457,7 @@ main(int argc, char *argv[]) my_obj_s += n_obj_incr; k++; - } while (curr_total_obj < n_obj); + } while (total_object_count < n_obj); if (my_rank == 0) { printf("Final Report: \n"); From ac3236967243a0143b4e828f157ed5d29af2e276 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 21:45:08 -0500 Subject: [PATCH 054/216] update output for uint64_t --- src/tests/kvtag_add_get_benchmark.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 9f39293e6..225d22d92 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -466,9 +466,9 @@ main(int argc, char *argv[]) printf("[Final Report 2] Iterations: %" PRIu64 " , Objects: %" PRIu64 " , Tags/Object: %" PRIu64 " , Queries/Iteration: " "%" PRIu64 " , \n", - k, curr_total_obj, n_attr, n_query); + k, total_object_count, n_attr, n_query); printf("[Final Report 3] Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", - (double)curr_total_obj / total_object_time, (double)(curr_total_obj * n_attr) / total_tag_time, + (double)total_object_count / total_object_time, (double)(total_object_count * n_attr) / total_tag_time, (double)(total_query_count * n_attr) / total_query_time); } From 94d8633081ab0130f490924760809c1d9de9d4b3 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 22:03:47 -0500 Subject: [PATCH 055/216] update output for uint64_t --- .../kvtag_add_get_benchmark/cori/template.sh | 6 ++--- scripts/kvtag_add_get_scale/cori/template.sh | 18 ++++++------- .../perlmutter/template.sh | 26 +++++++------------ 3 files changed, 21 insertions(+), 29 deletions(-) diff --git a/scripts/kvtag_add_get_benchmark/cori/template.sh b/scripts/kvtag_add_get_benchmark/cori/template.sh index 73eb670c5..dd3216c9c 100644 --- a/scripts/kvtag_add_get_benchmark/cori/template.sh +++ b/scripts/kvtag_add_get_benchmark/cori/template.sh @@ -1,7 +1,7 @@ #!/bin/bash -l -#REGSBATCH -p regular -#DBGSBATCH -p debug +#REGSBATCH -q regular +#DBGSBATCH -q debug #SBATCH -N NODENUM #REGSBATCH -t 4:00:00 #DBGSBATCH -t 0:30:00 @@ -16,7 +16,7 @@ # export PDC_DEBUG=0 -export PDC_TMPDIR=/global/cscratch1/sd/wzhang5/data/pdc/conf +export PDC_TMPDIR=$SCRATCH/data/pdc/conf rm -rf $PDC_TMPDIR/* diff --git a/scripts/kvtag_add_get_scale/cori/template.sh b/scripts/kvtag_add_get_scale/cori/template.sh index 73eb670c5..402c008ae 100644 --- a/scripts/kvtag_add_get_scale/cori/template.sh +++ b/scripts/kvtag_add_get_scale/cori/template.sh @@ -1,7 +1,7 @@ #!/bin/bash -l -#REGSBATCH -p regular -#DBGSBATCH -p debug +#REGSBATCH -q regular +#DBGSBATCH -q debug #SBATCH -N NODENUM #REGSBATCH -t 4:00:00 #DBGSBATCH -t 0:30:00 @@ -16,7 +16,7 @@ # export PDC_DEBUG=0 -export PDC_TMPDIR=/global/cscratch1/sd/wzhang5/data/pdc/conf +export PDC_TMPDIR=$SCRATCH/data/pdc/conf rm -rf $PDC_TMPDIR/* @@ -32,16 +32,14 @@ let TOTALPROC=$NCLIENT*$N_NODE EXECPATH=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/share/test/bin SERVER=$EXECPATH/pdc_server.exe -CLIENT=$EXECPATH/kvtag_add_get_benchmark +CLIENT=$EXECPATH/kvtag_add_get_scale CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* -MAX_OBJ_COUNT=$((1024*1024)) -OBJ_INCR=$((MAX_OBJ_COUNT/1024)) -ATTR_COUNT=ATTRNUM -ATTR_LENGTH=ATTRLEN -QUERY_COUNT=$((1024)) +NUM_OBJ=$((1024*1024)) +NUM_TAGS=$NUM_OBJ +NUM_QUERY=$((NUM_OBJ)) date @@ -57,7 +55,7 @@ sleep 5 echo "============================================" echo "KVTAGS with $N_NODE nodes" echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap stdbuf -i0 -o0 -e0 $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE +srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores --gres=craynetwork:1 --overlap stdbuf -i0 -o0 -e0 $CLIENT $NUM_OBJ $NUM_TAGS $NUM_QUERY echo "" echo "=================" diff --git a/scripts/kvtag_add_get_scale/perlmutter/template.sh b/scripts/kvtag_add_get_scale/perlmutter/template.sh index 56fa7cd02..423d59752 100644 --- a/scripts/kvtag_add_get_scale/perlmutter/template.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/template.sh @@ -1,22 +1,19 @@ #!/bin/bash -l -#REGSBATCH -p regular -#DBGSBATCH -p debug +#REGSBATCH -q regular +#DBGSBATCH -q debug #SBATCH -N NODENUM #REGSBATCH -t 4:00:00 #DBGSBATCH -t 0:30:00 -#SBATCH --gres=craynetwork:2 -#SBATCH -L SCRATCH -#SBATCH -C haswell -#SBATCH -J JOBNAME +#SBATCH -C cpu +#SBATCH -J JOBNAME #SBATCH -A m2621 #SBATCH -o o%j.JOBNAME.out #SBATCH -e o%j.JOBNAME.out - # export PDC_DEBUG=0 -export PDC_TMPDIR=/global/cscratch1/sd/wzhang5/data/pdc/conf +export PDC_TMPDIR=$SCRATCH/data/pdc/conf rm -rf $PDC_TMPDIR/* @@ -32,17 +29,14 @@ let TOTALPROC=$NCLIENT*$N_NODE EXECPATH=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/share/test/bin SERVER=$EXECPATH/pdc_server.exe -CLIENT=$EXECPATH/kvtag_add_get_benchmark +CLIENT=$EXECPATH/kvtag_add_get_scale CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* -NUM_OBJ= -MAX_OBJ_COUNT=$((1024*1024)) -OBJ_INCR=$((MAX_OBJ_COUNT/1024)) -ATTR_COUNT=ATTRNUM -ATTR_LENGTH=ATTRLEN -QUERY_COUNT=$((1024)) +NUM_OBJ=$((1024*1024)) +NUM_TAGS=$NUM_OBJ +NUM_QUERY=$((NUM_OBJ)) date @@ -58,7 +52,7 @@ sleep 5 echo "============================================" echo "KVTAGS with $N_NODE nodes" echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=256000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE +srun -N $N_NODE -n $TOTALPROC -c 2 --mem=256000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLIENT $NUM_OBJ $NUM_TAGS $NUM_QUERY echo "" echo "=================" From 653375bf592bad451da71ebb2bef66e94fb761c6 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 22:09:27 -0500 Subject: [PATCH 056/216] update scripts --- scripts/kvtag_add_get_benchmark/cori/clean.sh | 0 scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh | 0 scripts/kvtag_add_get_benchmark/cori/submit.sh | 0 scripts/kvtag_add_get_benchmark/cori/template.sh | 0 scripts/kvtag_add_get_scale/cori/clean.sh | 0 scripts/kvtag_add_get_scale/cori/gen_scripts.sh | 0 scripts/kvtag_add_get_scale/cori/submit.sh | 0 scripts/kvtag_add_get_scale/cori/template.sh | 0 scripts/kvtag_add_get_scale/perlmutter/clean.sh | 0 scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh | 0 scripts/kvtag_add_get_scale/perlmutter/submit.sh | 0 scripts/kvtag_add_get_scale/perlmutter/template.sh | 0 12 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/kvtag_add_get_benchmark/cori/clean.sh mode change 100644 => 100755 scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh mode change 100644 => 100755 scripts/kvtag_add_get_benchmark/cori/submit.sh mode change 100644 => 100755 scripts/kvtag_add_get_benchmark/cori/template.sh mode change 100644 => 100755 scripts/kvtag_add_get_scale/cori/clean.sh mode change 100644 => 100755 scripts/kvtag_add_get_scale/cori/gen_scripts.sh mode change 100644 => 100755 scripts/kvtag_add_get_scale/cori/submit.sh mode change 100644 => 100755 scripts/kvtag_add_get_scale/cori/template.sh mode change 100644 => 100755 scripts/kvtag_add_get_scale/perlmutter/clean.sh mode change 100644 => 100755 scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh mode change 100644 => 100755 scripts/kvtag_add_get_scale/perlmutter/submit.sh mode change 100644 => 100755 scripts/kvtag_add_get_scale/perlmutter/template.sh diff --git a/scripts/kvtag_add_get_benchmark/cori/clean.sh b/scripts/kvtag_add_get_benchmark/cori/clean.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh b/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_benchmark/cori/submit.sh b/scripts/kvtag_add_get_benchmark/cori/submit.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_benchmark/cori/template.sh b/scripts/kvtag_add_get_benchmark/cori/template.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_scale/cori/clean.sh b/scripts/kvtag_add_get_scale/cori/clean.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_scale/cori/gen_scripts.sh b/scripts/kvtag_add_get_scale/cori/gen_scripts.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_scale/cori/submit.sh b/scripts/kvtag_add_get_scale/cori/submit.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_scale/cori/template.sh b/scripts/kvtag_add_get_scale/cori/template.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_scale/perlmutter/clean.sh b/scripts/kvtag_add_get_scale/perlmutter/clean.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh b/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_scale/perlmutter/submit.sh b/scripts/kvtag_add_get_scale/perlmutter/submit.sh old mode 100644 new mode 100755 diff --git a/scripts/kvtag_add_get_scale/perlmutter/template.sh b/scripts/kvtag_add_get_scale/perlmutter/template.sh old mode 100644 new mode 100755 From ebe99ef248ec9bd1078ddff515be90b011b5ff58 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 22:29:19 -0500 Subject: [PATCH 057/216] update scripts --- scripts/kvtag_add_get_scale/cori/template.sh | 2 +- scripts/kvtag_add_get_scale/perlmutter/template.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/kvtag_add_get_scale/cori/template.sh b/scripts/kvtag_add_get_scale/cori/template.sh index 402c008ae..40a4dca34 100755 --- a/scripts/kvtag_add_get_scale/cori/template.sh +++ b/scripts/kvtag_add_get_scale/cori/template.sh @@ -37,7 +37,7 @@ CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* -NUM_OBJ=$((1024*1024)) +NUM_OBJ=$((1024*1024*100)) NUM_TAGS=$NUM_OBJ NUM_QUERY=$((NUM_OBJ)) diff --git a/scripts/kvtag_add_get_scale/perlmutter/template.sh b/scripts/kvtag_add_get_scale/perlmutter/template.sh index 423d59752..8b424ef88 100755 --- a/scripts/kvtag_add_get_scale/perlmutter/template.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/template.sh @@ -27,14 +27,14 @@ mkdir -p $PDC_TMPDIR let TOTALPROC=$NCLIENT*$N_NODE -EXECPATH=/global/cfs/cdirs/m2621/wzhang5/cori/install/pdc/share/test/bin +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin SERVER=$EXECPATH/pdc_server.exe CLIENT=$EXECPATH/kvtag_add_get_scale CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* -NUM_OBJ=$((1024*1024)) +NUM_OBJ=$((1024*1024*100)) NUM_TAGS=$NUM_OBJ NUM_QUERY=$((NUM_OBJ)) From aaa0b84632e4031ffd056ecc459b610b72da7df8 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 22:32:48 -0500 Subject: [PATCH 058/216] delete debugging message --- src/api/pdc_client_connect.c | 2 +- src/server/pdc_server_metadata.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index 897379e77..457001079 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -6961,7 +6961,7 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) } //TODO: delete this line after debugging. - printf("==CLIENT[%d]: PDC_add_kvtag::in.obj_id = %llu \n ", pdc_client_mpi_rank_g, in.obj_id); + // printf("==CLIENT[%d]: PDC_add_kvtag::in.obj_id = %llu \n ", pdc_client_mpi_rank_g, in.obj_id); server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); diff --git a/src/server/pdc_server_metadata.c b/src/server/pdc_server_metadata.c index 79cd22dfd..2331df3b1 100644 --- a/src/server/pdc_server_metadata.c +++ b/src/server/pdc_server_metadata.c @@ -2555,7 +2555,7 @@ PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out) hash_key = in->hash_value; obj_id = in->obj_id; - printf("==SERVER[%d]: PDC_add_kvtag::in.obj_id = %llu \n ", pdc_server_rank_g, obj_id); + // printf("==SERVER[%d]: PDC_add_kvtag::in.obj_id = %llu \n ", pdc_server_rank_g, obj_id); #ifdef ENABLE_MULTITHREAD // Obtain lock for hash table From ed0587870adf6886e5ed84d051967470e1589269 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 22:37:22 -0500 Subject: [PATCH 059/216] make Cmake to publish scripts directory --- CMakeLists.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c661eaa9..0922d44c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -455,6 +455,17 @@ install( PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) + +install( + DIRECTORY + ${PDC_SOURCE_DIR}/scripts + DESTINATION + ${PDC_BINARY_DIR} + FILES_MATCHING PATTERN "*.sh" + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE +) + + #install( # FILES # ${PDC_BINARY_DIR}/bin/pdc_server.exe From 0f3db861fbfed610925fec6fe6770f7a48a5ccd3 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 22:42:53 -0500 Subject: [PATCH 060/216] make Cmake to publish scripts directory --- CMakeLists.txt | 2 +- scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh | 2 +- scripts/kvtag_add_get_scale/cori/gen_scripts.sh | 2 +- scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0922d44c9..446f1d024 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -460,7 +460,7 @@ install( DIRECTORY ${PDC_SOURCE_DIR}/scripts DESTINATION - ${PDC_BINARY_DIR} + ${PDC_INSTALL_DATA_DIR} FILES_MATCHING PATTERN "*.sh" PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) diff --git a/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh b/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh index 54606da2b..62eb1a2b2 100755 --- a/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh +++ b/scripts/kvtag_add_get_benchmark/cori/gen_scripts.sh @@ -9,7 +9,7 @@ for (( i = 1; i <= $MAX_NODE; i*=2 )); do for (( j = 1; j <= $MAX_ATTR; j*=4 )); do for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/$JOBNAME.sh + TARGET=./$i/$JOBNAME.sbatch cp template.sh $TARGET sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET sed -i "s/NODENUM/${i}/g" $TARGET diff --git a/scripts/kvtag_add_get_scale/cori/gen_scripts.sh b/scripts/kvtag_add_get_scale/cori/gen_scripts.sh index 54606da2b..62eb1a2b2 100755 --- a/scripts/kvtag_add_get_scale/cori/gen_scripts.sh +++ b/scripts/kvtag_add_get_scale/cori/gen_scripts.sh @@ -9,7 +9,7 @@ for (( i = 1; i <= $MAX_NODE; i*=2 )); do for (( j = 1; j <= $MAX_ATTR; j*=4 )); do for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/$JOBNAME.sh + TARGET=./$i/$JOBNAME.sbatch cp template.sh $TARGET sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET sed -i "s/NODENUM/${i}/g" $TARGET diff --git a/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh b/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh index 54606da2b..62eb1a2b2 100755 --- a/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh @@ -9,7 +9,7 @@ for (( i = 1; i <= $MAX_NODE; i*=2 )); do for (( j = 1; j <= $MAX_ATTR; j*=4 )); do for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/$JOBNAME.sh + TARGET=./$i/$JOBNAME.sbatch cp template.sh $TARGET sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET sed -i "s/NODENUM/${i}/g" $TARGET From 5552ad06c0a4ba1326730c9f6b94c122a289f6db Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 22:45:48 -0500 Subject: [PATCH 061/216] make Cmake to publish scripts directory --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 446f1d024..2e353dbc2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -460,7 +460,7 @@ install( DIRECTORY ${PDC_SOURCE_DIR}/scripts DESTINATION - ${PDC_INSTALL_DATA_DIR} + ${CMAKE_INSTALL_PREFIX} FILES_MATCHING PATTERN "*.sh" PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) From 0e63480cc3ba72f88c6596b3ebf5b588ebab1002 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 22:47:44 -0500 Subject: [PATCH 062/216] make Cmake to publish scripts directory --- .../perlmutter/clean.sh | 8 ++ .../perlmutter/gen_scripts.sh | 25 ++++++ .../perlmutter/submit.sh | 78 +++++++++++++++++++ .../perlmutter/template.sh | 66 ++++++++++++++++ 4 files changed, 177 insertions(+) create mode 100755 scripts/kvtag_add_get_benchmark/perlmutter/clean.sh create mode 100755 scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh create mode 100755 scripts/kvtag_add_get_benchmark/perlmutter/submit.sh create mode 100755 scripts/kvtag_add_get_benchmark/perlmutter/template.sh diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/clean.sh b/scripts/kvtag_add_get_benchmark/perlmutter/clean.sh new file mode 100755 index 000000000..d6476962e --- /dev/null +++ b/scripts/kvtag_add_get_benchmark/perlmutter/clean.sh @@ -0,0 +1,8 @@ +#!/bin/bash +MAX_NODE=512 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + + rm -rf $i/* + +done \ No newline at end of file diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh b/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh new file mode 100755 index 000000000..62eb1a2b2 --- /dev/null +++ b/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh @@ -0,0 +1,25 @@ +#!/bin/bash +N_THREAD=NO +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + mkdir -p $i + for (( j = 1; j <= $MAX_ATTR; j*=4 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + JOBNAME=kvtag_bench_${i}_${j}_${k} + TARGET=./$i/$JOBNAME.sbatch + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/ATTRNUM/${j}/g" $TARGET + sed -i "s/ATTRLEN/${k}/g" $TARGET + if [[ "$i" -gt "16" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi + done + done +done diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/submit.sh b/scripts/kvtag_add_get_benchmark/perlmutter/submit.sh new file mode 100755 index 000000000..2ca6badf8 --- /dev/null +++ b/scripts/kvtag_add_get_benchmark/perlmutter/submit.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# MIN_PROC=4 +# MAX_PROC=128 +MIN_PROC=1 +MAX_PROC=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +curdir=$(pwd) + +first_submit=1 + +for (( i = 1; i <= $MAX_PROC; i*=2 )); do + mkdir -p $i + for (( j = 1; j <= $MAX_ATTR; j*=4 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + JOBNAME=kvtag_bench_${i}_${j}_${k} + TARGET=./$i/JOBNAME.sh + + njob=`squeue -u $USER | grep kvtag_bench | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep kvtag_bench | wc -l` + echo $njob + done + + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi + + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime + done + done +done + + +# for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do + +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# while [ $njob -ge 4 ] +# do +# sleeptime=$[ ( $RANDOM % 1000 ) ] +# sleep $sleeptime +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# done + + +# cd $curdir/$j +# for filename in ./*.sh ; do + +# if [[ $first_submit == 1 ]]; then +# # Submit first job w/o dependency +# echo "Submitting $filename" +# job=`sbatch $filename` +# first_submit=0 +# else +# echo "Submitting $filename after ${job: -8}" +# job=`sbatch -d afterany:${job: -8} $filename` +# fi + +# sleeptime=$[ ( $RANDOM % 5 ) ] +# sleep $sleeptime + +# done +# done diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh new file mode 100755 index 000000000..fc17ed525 --- /dev/null +++ b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh @@ -0,0 +1,66 @@ +#!/bin/bash -l + +#REGSBATCH -q regular +#DBGSBATCH -q debug +#SBATCH -N NODENUM +#REGSBATCH -t 4:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH -C cpu +#SBATCH -J JOBNAME +#SBATCH -A m2621 +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + + +# export PDC_DEBUG=0 + +export PDC_TMPDIR=$SCRATCH/data/pdc/conf + +rm -rf $PDC_TMPDIR/* + +REPEAT=1 + +N_NODE=NODENUM +NCLIENT=31 + +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE +mkdir -p $PDC_TMPDIR + +let TOTALPROC=$NCLIENT*$N_NODE + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$EXECPATH/kvtag_add_get_benchmark +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* + +MAX_OBJ_COUNT=$((1024*1024)) +OBJ_INCR=$((MAX_OBJ_COUNT/1024)) +ATTR_COUNT=ATTRNUM +ATTR_LENGTH=ATTRLEN +QUERY_COUNT=$((1024)) + +date + + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +srun -N $N_NODE -n $N_NODE -c 2 --mem=100000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLOSE + +date From fd745f8092d5295a42c0f4556a585a4a86717638 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 23:16:47 -0500 Subject: [PATCH 063/216] make Cmake to publish scripts directory --- src/tests/kvtag_add_get_benchmark.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 225d22d92..d3550847a 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -430,7 +430,11 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif - + if (my_rank == 0) { + printf("send queries\n"); + fflush(stdout); + } + send_queries(my_obj_s, my_query, n_attr, obj_ids, query_rst_cache, value_size); #ifdef ENABLE_MPI From 53e0a156839837c2e08b82aa15a9567a9878f3ac Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 23:24:35 -0500 Subject: [PATCH 064/216] update tag names --- src/tests/kvtag_add_get_benchmark.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index d3550847a..fbfd8ca7a 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -242,7 +242,7 @@ get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, uint64_t n_attr, void **tag char tag_name[128]; for (i = 0; i < n_attr; i++) { - sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", obj_name_v, i); + sprintf(tag_name, "tag%llu.%llu", obj_name_v, i); if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], (void *)&value_size[i]) < 0) printf("fail to get a kvtag from o%" PRIu64 "\n", obj_name_v); } From 4e6eb2700fa3b987720ee621efcc768ce4362026 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 5 Apr 2023 23:27:51 -0500 Subject: [PATCH 065/216] update tag names --- src/tests/kvtag_add_get_benchmark.c | 35 +++++++++++++---------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index fbfd8ca7a..7ac9c5c09 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -242,7 +242,7 @@ get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, uint64_t n_attr, void **tag char tag_name[128]; for (i = 0; i < n_attr; i++) { - sprintf(tag_name, "tag%llu.%llu", obj_name_v, i); + sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", obj_name_v, i); if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], (void *)&value_size[i]) < 0) printf("fail to get a kvtag from o%" PRIu64 "\n", obj_name_v); } @@ -323,18 +323,18 @@ closePDC(pdcid_t pdc, pdcid_t cont_prop, pdcid_t cont, pdcid_t obj_prop) int main(int argc, char *argv[]) { - pdcid_t pdc, cont_prop, cont, obj_prop; - pdcid_t * obj_ids; - uint64_t n_obj, n_obj_incr, my_obj, my_obj_s; - uint64_t n_attr, n_attr_len, n_query, my_query, my_query_s; - uint64_t n_servers, n_clients; - uint64_t i, k; - int proc_num, my_rank; - double stime = 0.0, step_elapse = 0.0; - double total_object_time = 0.0, total_tag_time = 0.0, total_query_time = 0.0; - uint64_t total_object_count = 0, total_tag_count = 0, total_query_count = 0; - void ** query_rst_cache; - uint64_t * value_size; + pdcid_t pdc, cont_prop, cont, obj_prop; + pdcid_t * obj_ids; + uint64_t n_obj, n_obj_incr, my_obj, my_obj_s; + uint64_t n_attr, n_attr_len, n_query, my_query, my_query_s; + uint64_t n_servers, n_clients; + uint64_t i, k; + int proc_num, my_rank; + double stime = 0.0, step_elapse = 0.0; + double total_object_time = 0.0, total_tag_time = 0.0, total_query_time = 0.0; + uint64_t total_object_count = 0, total_tag_count = 0, total_query_count = 0; + void ** query_rst_cache; + uint64_t *value_size; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); @@ -430,11 +430,7 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif - if (my_rank == 0) { - printf("send queries\n"); - fflush(stdout); - } - + send_queries(my_obj_s, my_query, n_attr, obj_ids, query_rst_cache, value_size); #ifdef ENABLE_MPI @@ -472,7 +468,8 @@ main(int argc, char *argv[]) "%" PRIu64 " , \n", k, total_object_count, n_attr, n_query); printf("[Final Report 3] Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", - (double)total_object_count / total_object_time, (double)(total_object_count * n_attr) / total_tag_time, + (double)total_object_count / total_object_time, + (double)(total_object_count * n_attr) / total_tag_time, (double)(total_query_count * n_attr) / total_query_time); } From 71a6a9a587f41c3a5aaca3073847735bfdcc2e39 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 6 Apr 2023 00:40:56 -0500 Subject: [PATCH 066/216] update query startingpos --- src/tests/kvtag_add_get_benchmark.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 7ac9c5c09..b96b981f5 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -431,7 +431,7 @@ main(int argc, char *argv[]) stime = MPI_Wtime(); #endif - send_queries(my_obj_s, my_query, n_attr, obj_ids, query_rst_cache, value_size); + send_queries(my_query_s, my_query, n_attr, obj_ids, query_rst_cache, value_size); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); From fed2e60adcd837ebf673c3bd2c6f6d13c81db23b Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 6 Apr 2023 09:47:25 -0500 Subject: [PATCH 067/216] update query startingpos --- src/tests/kvtag_add_get_benchmark.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index b96b981f5..1b5f26815 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -188,7 +188,7 @@ void create_object(uint64_t my_obj, uint64_t my_obj_s, pdcid_t cont, pdcid_t obj_prop, pdcid_t *obj_ids) { uint64_t i, v; - char obj_name[128]; + char obj_name[256]; for (i = 0; i < my_obj; i++) { v = i + my_obj_s; @@ -214,12 +214,12 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value pdcid_t *obj_ids) { uint64_t i, j, v; - char tag_name[128]; + char tag_name[256]; for (i = 0; i < my_obj; i++) { v = i + my_obj_s; for (j = 0; j < n_attr; j++) { - sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", v, i); + sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", v, j); if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], tag_value_len + 1) < 0) printf("fail to add a kvtag to o%" PRIu64 "\n", v); } @@ -239,7 +239,7 @@ void get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, uint64_t n_attr, void **tag_values, uint64_t *value_size) { uint64_t i; - char tag_name[128]; + char tag_name[256]; for (i = 0; i < n_attr; i++) { sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", obj_name_v, i); @@ -267,10 +267,16 @@ void send_queries(uint64_t my_obj_s, uint64_t n_query, uint64_t n_attr, pdcid_t *obj_ids, void **tag_values, uint64_t *value_size) { - uint64_t i, v; + uint64_t i, j, v; + char tag_name[128]; for (i = 0; i < n_query; i++) { v = i + my_obj_s; + // for (j = 0; j < n_attr; j++) { + // sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", v, j); + // if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[j], (void *)&value_size[j]) < 0) + // printf("fail to get a kvtag from o%" PRIu64 "\n", v); + // } get_object_tags(obj_ids[i], v, n_attr, &tag_values[i * n_attr], &value_size[i * n_attr]); } } @@ -360,6 +366,13 @@ main(int argc, char *argv[]) goto done; } + if (n_query > n_obj_incr) { + if (my_rank == 0) { + printf("n_query cannot be larger than n_obj_incr! Exiting...\n"); + } + goto done; + } + if (my_rank == 0) printf("Create %" PRIu64 " obj, %" PRIu64 " tags, query %" PRIu64 "\n", n_obj, n_attr, n_query); @@ -455,6 +468,7 @@ main(int argc, char *argv[]) fflush(stdout); my_obj_s += n_obj_incr; + my_query_s += n_obj_incr; k++; } while (total_object_count < n_obj); From 591e5c7c8a436ece86712298261330d276d7ae40 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 6 Apr 2023 11:16:13 -0500 Subject: [PATCH 068/216] update job scripts --- scripts/kvtag_add_get_benchmark/cori/template.sh | 4 ++-- scripts/kvtag_add_get_benchmark/perlmutter/template.sh | 4 ++-- src/tests/kvtag_add_get_benchmark.c | 8 +------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/scripts/kvtag_add_get_benchmark/cori/template.sh b/scripts/kvtag_add_get_benchmark/cori/template.sh index dd3216c9c..1db2938f3 100755 --- a/scripts/kvtag_add_get_benchmark/cori/template.sh +++ b/scripts/kvtag_add_get_benchmark/cori/template.sh @@ -37,11 +37,11 @@ CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* -MAX_OBJ_COUNT=$((1024*1024)) +MAX_OBJ_COUNT=$((1024*1024*1024)) OBJ_INCR=$((MAX_OBJ_COUNT/1024)) ATTR_COUNT=ATTRNUM ATTR_LENGTH=ATTRLEN -QUERY_COUNT=$((1024)) +QUERY_COUNT=$((OBJ_INCR)) date diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh index fc17ed525..9c69b9872 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh @@ -35,11 +35,11 @@ CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* -MAX_OBJ_COUNT=$((1024*1024)) +MAX_OBJ_COUNT=$((1024*1024*1024)) OBJ_INCR=$((MAX_OBJ_COUNT/1024)) ATTR_COUNT=ATTRNUM ATTR_LENGTH=ATTRLEN -QUERY_COUNT=$((1024)) +QUERY_COUNT=$((OBJ_INCR)) date diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 1b5f26815..c0801135e 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -267,16 +267,10 @@ void send_queries(uint64_t my_obj_s, uint64_t n_query, uint64_t n_attr, pdcid_t *obj_ids, void **tag_values, uint64_t *value_size) { - uint64_t i, j, v; - char tag_name[128]; + uint64_t i, v; for (i = 0; i < n_query; i++) { v = i + my_obj_s; - // for (j = 0; j < n_attr; j++) { - // sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", v, j); - // if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[j], (void *)&value_size[j]) < 0) - // printf("fail to get a kvtag from o%" PRIu64 "\n", v); - // } get_object_tags(obj_ids[i], v, n_attr, &tag_values[i * n_attr], &value_size[i * n_attr]); } } From 9b88faec1a7fecae7a585e8fae87ba132a0da25c Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 6 Apr 2023 16:20:15 -0500 Subject: [PATCH 069/216] add progressive timing for kvtag_add_get_scale --- src/tests/kvtag_add_get_scale.c | 60 +++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index 95a32591a..28f2352ea 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -70,9 +70,10 @@ main(int argc, char *argv[]) pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t * obj_ids; int n_obj, n_add_tag, n_query, my_obj, my_obj_s, my_add_tag, my_query, my_add_tag_s, my_query_s; + int obj_10percent = 0, tag_10percent = 0, query_10percent = 0; int proc_num, my_rank, i, v; char obj_name[128]; - double stime, total_time; + double stime, total_time, percent_time; pdc_kvtag_t kvtag; void ** values; size_t value_size; @@ -100,6 +101,10 @@ main(int argc, char *argv[]) assign_work_to_rank(my_rank, proc_num, n_query, &my_query, &my_query_s); assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); + obj_10percent = my_obj / 10; + tag_10percent = my_add_tag / 10; + query_10percent = my_query / 10; + if (my_rank == 0) printf("Create %d obj, %d tags, query %d\n", my_obj, my_add_tag, my_query); @@ -123,15 +128,40 @@ main(int argc, char *argv[]) // Create a number of objects, add at least one tag to that object obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + for (i = 0; i < my_obj; i++) { sprintf(obj_name, "obj%d", my_obj_s + i); obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); if (obj_ids[i] <= 0) printf("Fail to create object @ line %d!\n", __LINE__); + + if (i % obj_10percent == 0) { +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / obj_10percent; + int estimated_current_object_number = n_obj / 100 * current_percentage; + double tps = estimated_current_object_number / percent_time; + printf("[OBJ PROGRESS %d%% ] %d objects, %.2f seconds, TPS: %.4f", current_percentage, + estimated_current_object_number, percent_time, tps); + } +#endif + } } +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) - printf("Created %d objects\n", n_obj); + printf("Total time to create %d objects: %.4f\n\n", n_obj, total_time); // Add tags kvtag.name = "Group"; @@ -146,6 +176,18 @@ main(int argc, char *argv[]) v = i + my_add_tag_s; if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) printf("fail to add a kvtag to o%d\n", i + my_obj_s); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / tag_10percent; + int estimated_current_tag_number = n_obj / 100 * current_percentage; + double tps = estimated_current_tag_number / percent_time; + printf("[TAG PROGRESS %d%% ] %d tags, %.2f seconds, TPS: %.4f", current_percentage, + estimated_current_tag_number, percent_time, tps); + } +#endif } #ifdef ENABLE_MPI @@ -164,6 +206,18 @@ main(int argc, char *argv[]) for (i = 0; i < my_query; i++) { if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_size) < 0) printf("fail to get a kvtag from o%d\n", i + my_query_s); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / query_10percent; + int estimated_current_query_number = n_obj / 100 * current_percentage; + double tps = estimated_current_query_number / percent_time; + printf("[QRY PROGRESS %d%% ] %d queries, %.2f seconds, TPS: %.4f", current_percentage, + estimated_current_query_number, percent_time, tps); + } +#endif } #ifdef ENABLE_MPI @@ -171,7 +225,7 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to retrieve tags from %d objects: %.4f\n", n_query, total_time); + printf("Total time to retrieve 1 tag from %d objects: %.4f\n", n_query, total_time); fflush(stdout); From c20956d01ff6023080abb5af24fc8d9f93d93f7b Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 6 Apr 2023 16:38:28 -0500 Subject: [PATCH 070/216] fix iteration count in final report --- src/tests/kvtag_add_get_benchmark.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index c0801135e..27e0836ad 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -474,7 +474,7 @@ main(int argc, char *argv[]) printf("[Final Report 2] Iterations: %" PRIu64 " , Objects: %" PRIu64 " , Tags/Object: %" PRIu64 " , Queries/Iteration: " "%" PRIu64 " , \n", - k, total_object_count, n_attr, n_query); + k-1, total_object_count, n_attr, n_query); printf("[Final Report 3] Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", (double)total_object_count / total_object_time, (double)(total_object_count * n_attr) / total_tag_time, From eab118697b3a0258605d24ce6d575d7a26c0a193 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 7 Apr 2023 13:02:37 -0500 Subject: [PATCH 071/216] update job scripts and benckmark program --- .../kvtag_add_get_scale/cori/gen_scripts.sh | 26 ++++------ scripts/kvtag_add_get_scale/cori/submit.sh | 48 +++++++++---------- .../perlmutter/gen_scripts.sh | 26 ++++------ .../kvtag_add_get_scale/perlmutter/submit.sh | 48 +++++++++---------- src/tests/kvtag_add_get_benchmark.c | 10 ++-- 5 files changed, 69 insertions(+), 89 deletions(-) diff --git a/scripts/kvtag_add_get_scale/cori/gen_scripts.sh b/scripts/kvtag_add_get_scale/cori/gen_scripts.sh index 62eb1a2b2..e8ee7b11d 100755 --- a/scripts/kvtag_add_get_scale/cori/gen_scripts.sh +++ b/scripts/kvtag_add_get_scale/cori/gen_scripts.sh @@ -6,20 +6,14 @@ MAX_ATTRLEN=1000 for (( i = 1; i <= $MAX_NODE; i*=2 )); do mkdir -p $i - for (( j = 1; j <= $MAX_ATTR; j*=4 )); do - for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do - JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/$JOBNAME.sbatch - cp template.sh $TARGET - sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET - sed -i "s/NODENUM/${i}/g" $TARGET - sed -i "s/ATTRNUM/${j}/g" $TARGET - sed -i "s/ATTRLEN/${k}/g" $TARGET - if [[ "$i" -gt "16" ]]; then - sed -i "s/REG//g" $TARGET - else - sed -i "s/DBG//g" $TARGET - fi - done - done + JOBNAME=kvtag_scale${i} + TARGET=./$i/$JOBNAME.sbatch + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + if [[ "$i" -gt "16" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi done diff --git a/scripts/kvtag_add_get_scale/cori/submit.sh b/scripts/kvtag_add_get_scale/cori/submit.sh index 2ca6badf8..192522e31 100755 --- a/scripts/kvtag_add_get_scale/cori/submit.sh +++ b/scripts/kvtag_add_get_scale/cori/submit.sh @@ -13,35 +13,31 @@ first_submit=1 for (( i = 1; i <= $MAX_PROC; i*=2 )); do mkdir -p $i - for (( j = 1; j <= $MAX_ATTR; j*=4 )); do - for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do - JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/JOBNAME.sh + JOBNAME=kvtag_scale_${i} + TARGET=./$i/JOBNAME.sh - njob=`squeue -u $USER | grep kvtag_bench | wc -l` - echo $njob - while [ $njob -ge 4 ] - do - sleeptime=$[ ( $RANDOM % 1000 ) ] - sleep $sleeptime - njob=`squeue -u $USER | grep kvtag_bench | wc -l` - echo $njob - done + njob=`squeue -u $USER | grep kvtag_scale | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep kvtag_scale | wc -l` + echo $njob + done - if [[ $first_submit == 1 ]]; then - # Submit first job w/o dependency - echo "Submitting $TARGET" - job=`sbatch $TARGET` - first_submit=0 - else - echo "Submitting $TARGET after ${job: -8}" - job=`sbatch -d afterany:${job: -8} $TARGET` - fi + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi - sleeptime=$[ ( $RANDOM % 5 ) ] - sleep $sleeptime - done - done + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime done diff --git a/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh b/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh index 62eb1a2b2..4af771821 100755 --- a/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh @@ -6,20 +6,14 @@ MAX_ATTRLEN=1000 for (( i = 1; i <= $MAX_NODE; i*=2 )); do mkdir -p $i - for (( j = 1; j <= $MAX_ATTR; j*=4 )); do - for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do - JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/$JOBNAME.sbatch - cp template.sh $TARGET - sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET - sed -i "s/NODENUM/${i}/g" $TARGET - sed -i "s/ATTRNUM/${j}/g" $TARGET - sed -i "s/ATTRLEN/${k}/g" $TARGET - if [[ "$i" -gt "16" ]]; then - sed -i "s/REG//g" $TARGET - else - sed -i "s/DBG//g" $TARGET - fi - done - done + JOBNAME=kvtag_scale_${i} + TARGET=./$i/$JOBNAME.sbatch + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + if [[ "$i" -gt "16" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi done diff --git a/scripts/kvtag_add_get_scale/perlmutter/submit.sh b/scripts/kvtag_add_get_scale/perlmutter/submit.sh index 2ca6badf8..192522e31 100755 --- a/scripts/kvtag_add_get_scale/perlmutter/submit.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/submit.sh @@ -13,35 +13,31 @@ first_submit=1 for (( i = 1; i <= $MAX_PROC; i*=2 )); do mkdir -p $i - for (( j = 1; j <= $MAX_ATTR; j*=4 )); do - for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do - JOBNAME=kvtag_bench_${i}_${j}_${k} - TARGET=./$i/JOBNAME.sh + JOBNAME=kvtag_scale_${i} + TARGET=./$i/JOBNAME.sh - njob=`squeue -u $USER | grep kvtag_bench | wc -l` - echo $njob - while [ $njob -ge 4 ] - do - sleeptime=$[ ( $RANDOM % 1000 ) ] - sleep $sleeptime - njob=`squeue -u $USER | grep kvtag_bench | wc -l` - echo $njob - done + njob=`squeue -u $USER | grep kvtag_scale | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep kvtag_scale | wc -l` + echo $njob + done - if [[ $first_submit == 1 ]]; then - # Submit first job w/o dependency - echo "Submitting $TARGET" - job=`sbatch $TARGET` - first_submit=0 - else - echo "Submitting $TARGET after ${job: -8}" - job=`sbatch -d afterany:${job: -8} $TARGET` - fi + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi - sleeptime=$[ ( $RANDOM % 5 ) ] - sleep $sleeptime - done - done + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime done diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 27e0836ad..be332a265 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -328,7 +328,7 @@ main(int argc, char *argv[]) uint64_t n_obj, n_obj_incr, my_obj, my_obj_s; uint64_t n_attr, n_attr_len, n_query, my_query, my_query_s; uint64_t n_servers, n_clients; - uint64_t i, k; + uint64_t i = 0, k = 0; int proc_num, my_rank; double stime = 0.0, step_elapse = 0.0; double total_object_time = 0.0, total_tag_time = 0.0, total_query_time = 0.0; @@ -377,9 +377,9 @@ main(int argc, char *argv[]) char **tag_values = gen_strings(n_attr, n_attr_len); - k = 1; - do { + + k++; #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); @@ -463,7 +463,7 @@ main(int argc, char *argv[]) my_obj_s += n_obj_incr; my_query_s += n_obj_incr; - k++; + } while (total_object_count < n_obj); @@ -474,7 +474,7 @@ main(int argc, char *argv[]) printf("[Final Report 2] Iterations: %" PRIu64 " , Objects: %" PRIu64 " , Tags/Object: %" PRIu64 " , Queries/Iteration: " "%" PRIu64 " , \n", - k-1, total_object_count, n_attr, n_query); + k, total_object_count, n_attr, n_query); printf("[Final Report 3] Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", (double)total_object_count / total_object_time, (double)(total_object_count * n_attr) / total_tag_time, From fe93e8ec80d873c5fede2fc85f4eff2e5afc2c94 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 7 Apr 2023 13:15:05 -0500 Subject: [PATCH 072/216] update message format --- src/tests/kvtag_add_get_scale.c | 44 ++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index 28f2352ea..bfa949961 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -146,9 +146,9 @@ main(int argc, char *argv[]) percent_time = MPI_Wtime() - stime; if (my_rank == 0) { int current_percentage = i / obj_10percent; - int estimated_current_object_number = n_obj / 100 * current_percentage; + int estimated_current_object_number = n_obj / 1000 * current_percentage; double tps = estimated_current_object_number / percent_time; - printf("[OBJ PROGRESS %d%% ] %d objects, %.2f seconds, TPS: %.4f", current_percentage, + printf("[OBJ PROGRESS %d%% ] %d objects, %.2f seconds, TPS: %.4f \n", current_percentage, estimated_current_object_number, percent_time, tps); } #endif @@ -177,17 +177,19 @@ main(int argc, char *argv[]) if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) printf("fail to add a kvtag to o%d\n", i + my_obj_s); + if (i % tag_10percent == 0) { #ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - percent_time = MPI_Wtime() - stime; - if (my_rank == 0) { - int current_percentage = i / tag_10percent; - int estimated_current_tag_number = n_obj / 100 * current_percentage; - double tps = estimated_current_tag_number / percent_time; - printf("[TAG PROGRESS %d%% ] %d tags, %.2f seconds, TPS: %.4f", current_percentage, - estimated_current_tag_number, percent_time, tps); - } + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / tag_10percent; + int estimated_current_tag_number = n_obj / 1000 * current_percentage; + double tps = estimated_current_tag_number / percent_time; + printf("[TAG PROGRESS %d%% ] %d tags, %.2f seconds, TPS: %.4f \n", current_percentage, + estimated_current_tag_number, percent_time, tps); + } #endif + } } #ifdef ENABLE_MPI @@ -207,17 +209,19 @@ main(int argc, char *argv[]) if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_size) < 0) printf("fail to get a kvtag from o%d\n", i + my_query_s); + if (i % query_10percent == 0) { #ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - percent_time = MPI_Wtime() - stime; - if (my_rank == 0) { - int current_percentage = i / query_10percent; - int estimated_current_query_number = n_obj / 100 * current_percentage; - double tps = estimated_current_query_number / percent_time; - printf("[QRY PROGRESS %d%% ] %d queries, %.2f seconds, TPS: %.4f", current_percentage, - estimated_current_query_number, percent_time, tps); - } + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / query_10percent; + int estimated_current_query_number = n_obj / 1000 * current_percentage; + double tps = estimated_current_query_number / percent_time; + printf("[QRY PROGRESS %d%% ] %d queries, %.2f seconds, TPS: %.4f \n", current_percentage, + estimated_current_query_number, percent_time, tps); + } #endif + } } #ifdef ENABLE_MPI From 7f16ee0849cce6bd40dd0536bb360e880145a8c6 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 7 Apr 2023 13:19:46 -0500 Subject: [PATCH 073/216] update message format --- src/tests/kvtag_add_get_scale.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index bfa949961..dc939155f 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -145,8 +145,8 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { - int current_percentage = i / obj_10percent; - int estimated_current_object_number = n_obj / 1000 * current_percentage; + int current_percentage = i / obj_10percent * 10; + int estimated_current_object_number = n_obj / 100 * current_percentage; double tps = estimated_current_object_number / percent_time; printf("[OBJ PROGRESS %d%% ] %d objects, %.2f seconds, TPS: %.4f \n", current_percentage, estimated_current_object_number, percent_time, tps); @@ -182,8 +182,8 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { - int current_percentage = i / tag_10percent; - int estimated_current_tag_number = n_obj / 1000 * current_percentage; + int current_percentage = i / tag_10percent * 10; + int estimated_current_tag_number = n_obj / 100 * current_percentage; double tps = estimated_current_tag_number / percent_time; printf("[TAG PROGRESS %d%% ] %d tags, %.2f seconds, TPS: %.4f \n", current_percentage, estimated_current_tag_number, percent_time, tps); @@ -214,8 +214,8 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { - int current_percentage = i / query_10percent; - int estimated_current_query_number = n_obj / 1000 * current_percentage; + int current_percentage = i / query_10percent * 10; + int estimated_current_query_number = n_obj / 100 * current_percentage; double tps = estimated_current_query_number / percent_time; printf("[QRY PROGRESS %d%% ] %d queries, %.2f seconds, TPS: %.4f \n", current_percentage, estimated_current_query_number, percent_time, tps); From 0adf28e685f25d9e0990d4b922fa4188c7c5f84d Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 7 Apr 2023 13:25:28 -0500 Subject: [PATCH 074/216] update message format --- src/tests/kvtag_add_get_scale.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index dc939155f..2bc5cc94c 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -70,7 +70,7 @@ main(int argc, char *argv[]) pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t * obj_ids; int n_obj, n_add_tag, n_query, my_obj, my_obj_s, my_add_tag, my_query, my_add_tag_s, my_query_s; - int obj_10percent = 0, tag_10percent = 0, query_10percent = 0; + int obj_1percent = 0, tag_1percent = 0, query_1percent = 0; int proc_num, my_rank, i, v; char obj_name[128]; double stime, total_time, percent_time; @@ -101,9 +101,9 @@ main(int argc, char *argv[]) assign_work_to_rank(my_rank, proc_num, n_query, &my_query, &my_query_s); assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); - obj_10percent = my_obj / 10; - tag_10percent = my_add_tag / 10; - query_10percent = my_query / 10; + obj_1percent = my_obj / 100; + tag_1percent = my_add_tag / 100; + query_1percent = my_query / 100; if (my_rank == 0) printf("Create %d obj, %d tags, query %d\n", my_obj, my_add_tag, my_query); @@ -140,12 +140,12 @@ main(int argc, char *argv[]) if (obj_ids[i] <= 0) printf("Fail to create object @ line %d!\n", __LINE__); - if (i % obj_10percent == 0) { + if (i > 0 && i % obj_1percent == 0) { #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { - int current_percentage = i / obj_10percent * 10; + int current_percentage = i / obj_1percent; int estimated_current_object_number = n_obj / 100 * current_percentage; double tps = estimated_current_object_number / percent_time; printf("[OBJ PROGRESS %d%% ] %d objects, %.2f seconds, TPS: %.4f \n", current_percentage, @@ -161,7 +161,7 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) - printf("Total time to create %d objects: %.4f\n\n", n_obj, total_time); + printf("Total time to create %d objects: %.4f , throughput %.4f \n", n_obj, total_time, n_obj / total_time); // Add tags kvtag.name = "Group"; @@ -177,16 +177,16 @@ main(int argc, char *argv[]) if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) printf("fail to add a kvtag to o%d\n", i + my_obj_s); - if (i % tag_10percent == 0) { + if (i % tag_1percent == 0) { #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { - int current_percentage = i / tag_10percent * 10; + int current_percentage = i / tag_1percent; int estimated_current_tag_number = n_obj / 100 * current_percentage; double tps = estimated_current_tag_number / percent_time; printf("[TAG PROGRESS %d%% ] %d tags, %.2f seconds, TPS: %.4f \n", current_percentage, - estimated_current_tag_number, percent_time, tps); + estimated_current_tag_number, percent_time, tps); } #endif } @@ -197,7 +197,7 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to add tags to %d objects: %.4f\n", n_add_tag, total_time); + printf("Total time to add tags to %d objects: %.4f , throughput %.4f \n", n_add_tag, total_time, n_add_tag / total_time); values = (void **)calloc(my_query, sizeof(void *)); @@ -209,16 +209,16 @@ main(int argc, char *argv[]) if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_size) < 0) printf("fail to get a kvtag from o%d\n", i + my_query_s); - if (i % query_10percent == 0) { + if (i % query_1percent == 0) { #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { - int current_percentage = i / query_10percent * 10; + int current_percentage = i / query_1percent; int estimated_current_query_number = n_obj / 100 * current_percentage; double tps = estimated_current_query_number / percent_time; printf("[QRY PROGRESS %d%% ] %d queries, %.2f seconds, TPS: %.4f \n", current_percentage, - estimated_current_query_number, percent_time, tps); + estimated_current_query_number, percent_time, tps); } #endif } @@ -229,7 +229,7 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to retrieve 1 tag from %d objects: %.4f\n", n_query, total_time); + printf("Total time to retrieve 1 tag from %d objects: %.4f , throughput %.4f \n", n_query, total_time, n_query / total_time); fflush(stdout); From 8ebcf3b810b30d7488efe5b9f4aa868d4903c2b8 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 7 Apr 2023 13:28:02 -0500 Subject: [PATCH 075/216] update message format --- scripts/kvtag_add_get_scale/perlmutter/template.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/kvtag_add_get_scale/perlmutter/template.sh b/scripts/kvtag_add_get_scale/perlmutter/template.sh index 8b424ef88..d48cb29f6 100755 --- a/scripts/kvtag_add_get_scale/perlmutter/template.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/template.sh @@ -34,7 +34,7 @@ CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* -NUM_OBJ=$((1024*1024*100)) +NUM_OBJ=$((1024*1024*1024)) NUM_TAGS=$NUM_OBJ NUM_QUERY=$((NUM_OBJ)) From 41875be40f38b3dabe148988f95e76d056edbf83 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 7 Apr 2023 14:09:32 -0500 Subject: [PATCH 076/216] clang format --- src/api/pdc_client_connect.c | 2 +- src/tests/kvtag_add_get_benchmark.c | 3 +-- src/tests/kvtag_add_get_scale.c | 9 ++++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index 457001079..7293c6834 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -6960,7 +6960,7 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) in.hash_value = PDC_get_hash_by_name(cont_prop->cont_info_pub->name); } - //TODO: delete this line after debugging. + // TODO: delete this line after debugging. // printf("==CLIENT[%d]: PDC_add_kvtag::in.obj_id = %llu \n ", pdc_client_mpi_rank_g, in.obj_id); server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index be332a265..7b719ec96 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -378,7 +378,7 @@ main(int argc, char *argv[]) char **tag_values = gen_strings(n_attr, n_attr_len); do { - + k++; #ifdef ENABLE_MPI @@ -463,7 +463,6 @@ main(int argc, char *argv[]) my_obj_s += n_obj_incr; my_query_s += n_obj_incr; - } while (total_object_count < n_obj); diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index 2bc5cc94c..40b895b98 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -161,7 +161,8 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) - printf("Total time to create %d objects: %.4f , throughput %.4f \n", n_obj, total_time, n_obj / total_time); + printf("Total time to create %d objects: %.4f , throughput %.4f \n", n_obj, total_time, + n_obj / total_time); // Add tags kvtag.name = "Group"; @@ -197,7 +198,8 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to add tags to %d objects: %.4f , throughput %.4f \n", n_add_tag, total_time, n_add_tag / total_time); + printf("Total time to add tags to %d objects: %.4f , throughput %.4f \n", n_add_tag, total_time, + n_add_tag / total_time); values = (void **)calloc(my_query, sizeof(void *)); @@ -229,7 +231,8 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - printf("Total time to retrieve 1 tag from %d objects: %.4f , throughput %.4f \n", n_query, total_time, n_query / total_time); + printf("Total time to retrieve 1 tag from %d objects: %.4f , throughput %.4f \n", n_query, total_time, + n_query / total_time); fflush(stdout); From d294866eb085b4b6c3739b9585c40afa68ccae35 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 11:32:09 -0500 Subject: [PATCH 077/216] update job scripts --- scripts/kvtag_add_get_benchmark/perlmutter/template.sh | 7 ++++--- scripts/kvtag_add_get_scale/perlmutter/template.sh | 9 +++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh index 9c69b9872..a7e29d7db 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh @@ -21,6 +21,7 @@ rm -rf $PDC_TMPDIR/* REPEAT=1 N_NODE=NODENUM +# NCLIENT=127 NCLIENT=31 export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE @@ -48,19 +49,19 @@ echo "" echo "=============" echo "$i Init server" echo "=============" -srun -N $N_NODE -n $N_NODE -c 2 --mem=100000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $SERVER & +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*4)) -c 2 --cpu_bind=cores $SERVER & sleep 5 echo "============================================" echo "KVTAGS with $N_NODE nodes" echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE echo "" echo "=================" echo "$i Closing server" echo "=================" -srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLOSE +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE date diff --git a/scripts/kvtag_add_get_scale/perlmutter/template.sh b/scripts/kvtag_add_get_scale/perlmutter/template.sh index d48cb29f6..bec9b51e0 100755 --- a/scripts/kvtag_add_get_scale/perlmutter/template.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/template.sh @@ -3,7 +3,7 @@ #REGSBATCH -q regular #DBGSBATCH -q debug #SBATCH -N NODENUM -#REGSBATCH -t 4:00:00 +#REGSBATCH -t 1:00:00 #DBGSBATCH -t 0:30:00 #SBATCH -C cpu #SBATCH -J JOBNAME @@ -21,6 +21,7 @@ REPEAT=1 N_NODE=NODENUM NCLIENT=31 +# NCLIENT=126 export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE mkdir -p $PDC_TMPDIR @@ -45,19 +46,19 @@ echo "" echo "=============" echo "$i Init server" echo "=============" -srun -N $N_NODE -n $N_NODE -c 2 --mem=128000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $SERVER & +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*4)) -c 2 --cpu_bind=cores $SERVER & sleep 5 echo "============================================" echo "KVTAGS with $N_NODE nodes" echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=256000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLIENT $NUM_OBJ $NUM_TAGS $NUM_QUERY +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT $NUM_OBJ $NUM_TAGS $NUM_QUERY echo "" echo "=================" echo "$i Closing server" echo "=================" -srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLOSE +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE date From 68d4dab7bde4864f9dda34d3610ec3fa1cb85965 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 13:39:57 -0500 Subject: [PATCH 078/216] comment off object/container close procedure in benchmark to save node hours --- .../perlmutter/template.sh | 2 +- .../kvtag_add_get_scale/perlmutter/template.sh | 2 +- src/tests/kvtag_add_get_benchmark.c | 15 ++++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh index a7e29d7db..7ea2e6149 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh @@ -49,7 +49,7 @@ echo "" echo "=============" echo "$i Init server" echo "=============" -stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*4)) -c 2 --cpu_bind=cores $SERVER & +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & sleep 5 diff --git a/scripts/kvtag_add_get_scale/perlmutter/template.sh b/scripts/kvtag_add_get_scale/perlmutter/template.sh index bec9b51e0..392d498b8 100755 --- a/scripts/kvtag_add_get_scale/perlmutter/template.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/template.sh @@ -46,7 +46,7 @@ echo "" echo "=============" echo "$i Init server" echo "=============" -stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*4)) -c 2 --cpu_bind=cores $SERVER & +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & sleep 5 diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 7b719ec96..6c4457eb4 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -293,12 +293,12 @@ check_and_release_query_result(uint64_t n_query, uint64_t my_obj, uint64_t my_ob } } free(values); - // close objects - for (i = 0; i < my_obj; i++) { - v = i + my_obj_s; - if (PDCobj_close(obj_ids[i]) < 0) - printf("fail to close object o%" PRIu64 "\n", v); - } + // FIXME: close objects. This is currently commented off to save node hours for benchmarks. + // for (i = 0; i < my_obj; i++) { + // v = i + my_obj_s; + // if (PDCobj_close(obj_ids[i]) < 0) + // printf("fail to close object o%" PRIu64 "\n", v); + // } } void @@ -486,7 +486,8 @@ main(int argc, char *argv[]) free(tag_values); free(obj_ids); - closePDC(pdc, cont_prop, cont, obj_prop); + //FIXME: the following is currently commented off to reduce node hours taken by time-consuming resource releasing procedure. + // closePDC(pdc, cont_prop, cont, obj_prop); done: #ifdef ENABLE_MPI From bb6c87bac198c1dfbbffcf9c9929e368b68ccfde Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 13:43:11 -0500 Subject: [PATCH 079/216] change the max number of object to 1M --- scripts/kvtag_add_get_benchmark/perlmutter/template.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh index 7ea2e6149..a2ef04329 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh @@ -36,7 +36,7 @@ CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* -MAX_OBJ_COUNT=$((1024*1024*1024)) +MAX_OBJ_COUNT=$((1024*1024)) OBJ_INCR=$((MAX_OBJ_COUNT/1024)) ATTR_COUNT=ATTRNUM ATTR_LENGTH=ATTRLEN From b09b94ebfc5fc0af102e59ba52eed3383991161a Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 14:09:15 -0500 Subject: [PATCH 080/216] change the max length of attribute value --- scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh | 2 +- scripts/kvtag_add_get_benchmark/perlmutter/template.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh b/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh index 62eb1a2b2..9804bfee5 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh @@ -7,7 +7,7 @@ MAX_ATTRLEN=1000 for (( i = 1; i <= $MAX_NODE; i*=2 )); do mkdir -p $i for (( j = 1; j <= $MAX_ATTR; j*=4 )); do - for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k+=100 )); do JOBNAME=kvtag_bench_${i}_${j}_${k} TARGET=./$i/$JOBNAME.sbatch cp template.sh $TARGET diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh index a2ef04329..db06915c0 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh @@ -3,7 +3,7 @@ #REGSBATCH -q regular #DBGSBATCH -q debug #SBATCH -N NODENUM -#REGSBATCH -t 4:00:00 +#REGSBATCH -t 1:00:00 #DBGSBATCH -t 0:30:00 #SBATCH -C cpu #SBATCH -J JOBNAME From 2ef74178b0e0abf96567c6557f564e1f0da55cb1 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 14:10:41 -0500 Subject: [PATCH 081/216] change the max length of attribute value --- scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh b/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh index 9804bfee5..1bc285fe7 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh @@ -2,12 +2,12 @@ N_THREAD=NO MAX_NODE=512 MAX_ATTR=1024 -MAX_ATTRLEN=1000 +MAX_ATTRLEN=1000000 for (( i = 1; i <= $MAX_NODE; i*=2 )); do mkdir -p $i for (( j = 1; j <= $MAX_ATTR; j*=4 )); do - for (( k = 100; k <= $MAX_ATTRLEN; k+=100 )); do + for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do JOBNAME=kvtag_bench_${i}_${j}_${k} TARGET=./$i/$JOBNAME.sbatch cp template.sh $TARGET From 2129f9655031310d6eccc6a1a3074f10686dc8df Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 23:16:11 -0500 Subject: [PATCH 082/216] llsm tiff import test --- tools/CMakeLists.txt | 9 +- tools/llsm/CMakeLists.txt | 5 - tools/llsm/parallelReadTiff.c | 1136 ++++++++++++++++++++------------- tools/llsm/parallelReadTiff.h | 22 + tools/llsm_importer.c | 47 ++ 5 files changed, 757 insertions(+), 462 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index e2a017bd4..9e4062959 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -54,7 +54,6 @@ endif() option(USE_SYSTEM_OPENMP "Use system-installed OpenMP." ON) if(USE_SYSTEM_OPENMP) find_package(OpenMP REQUIRED) - if(OPENMP_FOUND) set(OPENMP_LIBRARIES "${OpenMP_C_LIBRARIES}") else() @@ -65,12 +64,18 @@ endif() add_definitions(-DENABLE_MPI=1) add_library(cjson cjson/cJSON.c) -add_subdirectory(llsm) +add_library(llsm_tiff llsm/parallelReadTiff.c) +target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS}) +target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/llsm) +target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES}) + +# add_subdirectory(llsm) set(PROGRAMS pdc_ls pdc_import pdc_export + llsm_importer ) foreach(program ${PROGRAMS}) diff --git a/tools/llsm/CMakeLists.txt b/tools/llsm/CMakeLists.txt index a0ea9404d..e69de29bb 100644 --- a/tools/llsm/CMakeLists.txt +++ b/tools/llsm/CMakeLists.txt @@ -1,5 +0,0 @@ - -add_library(llsm_tiff parallelReadTiff.c) -target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS}) -target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES}) \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index fb4283e6a..ef417ee60 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -1,91 +1,76 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "tiffio.h" -#include "omp.h" -#include "mex.h" -//mex -v COPTIMFLAGS="-O3 -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-I/global/home/groups/software/sl-7.x86_64/modules/libtiff/4.1.0/libtiff/' '-L/global/home/groups/software/sl-7.x86_64/modules/libtiff/4.1.0/libtiff/' -ltiff parallelReadTiff.c -//mex COMPFLAGS='$COMPFLAGS /openmp' '-IC:\Program Files (x86)\tiff\include\' '-LC:\Program Files (x86)\tiff\lib\' -ltiffd.lib C:\Users\Matt\Documents\parallelTiff\main.cpp - -//libtiff 4.4.0 -//mex -v COPTIMFLAGS="-O3 -DNDEBUG" LDOPTIMFLAGS="-O3 -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-I/clusterfs/fiona/matthewmueller/software/tiff-4.4.0/include' '-L/clusterfs/fiona/matthewmueller/software/tiff-4.4.0/lib' -ltiff parallelReadTiff.c - -// Handle the tilde character in filenames on Linux/Mac -#ifndef _WIN32 -#include -char* expandTilde(char* path) { - wordexp_t expPath; - wordexp(path, &expPath, 0); - return expPath.we_wordv[0]; -} -#endif +#include "parallelReadTiff.h" -void DummyHandler(const char* module, const char* fmt, va_list ap) +void +DummyHandler(const char *module, const char *fmt, va_list ap) { // ignore errors and warnings } // Backup method in case there are errors reading strips -void readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char* fileName, void* tiff, uint64_t bits, uint64_t startSlice, uint8_t flipXY){ - int32_t numWorkers = omp_get_max_threads(); - int32_t batchSize = (z-1)/numWorkers+1; - uint64_t bytes = bits/8; +void +readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (z - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; int32_t w; - #pragma omp parallel for - for(w = 0; w < numWorkers; w++){ +#pragma omp parallel for + for (w = 0; w < numWorkers; w++) { - TIFF* tif = TIFFOpen(fileName, "r"); - if(!tif) mexErrMsgIdAndTxt("tiff:threadError","Thread %d: File \"%s\" cannot be opened\n",w,fileName); + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:threadError", "Thread %d: File \"%s\" cannot be opened\n", w, fileName); - void* buffer = malloc(x*bytes); - for(int64_t dir = startSlice+(w*batchSize); dir < startSlice+((w+1)*batchSize); dir++){ - if(dir>=z+startSlice) break; + void *buffer = malloc(x * bytes); + for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); dir++) { + if (dir >= z + startSlice) + break; int counter = 0; - while(!TIFFSetDirectory(tif, (uint64_t)dir) && counter<3){ - printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n",w,fileName,dir,counter+1); + while (!TIFFSetDirectory(tif, (uint64_t)dir) && counter < 3) { + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, dir, + counter + 1); counter++; } - for (int64_t i = 0; i < y; i++) - { + for (int64_t i = 0; i < y; i++) { TIFFReadScanline(tif, buffer, i, 0); - if(!flipXY){ - memcpy(tiff+((i*x)*bytes),buffer,x*bytes); + if (!flipXY) { + memcpy(tiff + ((i * x) * bytes), buffer, x * bytes); continue; } - //loading the data into a buffer - switch(bits){ + // loading the data into a buffer + switch (bits) { case 8: // Map Values to flip x and y for MATLAB - for(int64_t j = 0; j < x; j++){ - ((uint8_t*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((uint8_t*)buffer)[j]; + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint8_t *)buffer)[j]; } - break; + break; case 16: // Map Values to flip x and y for MATLAB - for(int64_t j = 0; j < x; j++){ - ((uint16_t*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((uint16_t*)buffer)[j]; + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint16_t *)buffer)[j]; } - break; + break; case 32: // Map Values to flip x and y for MATLAB - for(int64_t j = 0; j < x; j++){ - ((float*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((float*)buffer)[j]; + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((float *)buffer)[j]; } - break; + break; case 64: // Map Values to flip x and y for MATLAB - for(int64_t j = 0; j < x; j++){ - ((double*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((double*)buffer)[j]; + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((double *)buffer)[j]; } - break; + break; } } } @@ -94,113 +79,127 @@ void readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char* fileNam } } -void readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char* fileName, void* tiff, uint64_t bits, uint64_t startSlice, uint64_t stripSize, uint8_t flipXY){ - int32_t numWorkers = omp_get_max_threads(); - int32_t batchSize = (z-1)/numWorkers+1; - uint64_t bytes = bits/8; +void +readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (z - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; uint16_t compressed = 1; - TIFF* tif = TIFFOpen(fileName, "r"); + TIFF * tif = TIFFOpen(fileName, "r"); TIFFGetField(tif, TIFFTAG_COMPRESSION, &compressed); - - - int32_t w; uint8_t errBak = 0; - uint8_t err = 0; - char errString[10000]; - if(compressed > 1 || z < 32768){ + uint8_t err = 0; + char errString[10000]; + if (compressed > 1 || z < 32768) { TIFFClose(tif); - #pragma omp parallel for - for(w = 0; w < numWorkers; w++){ +#pragma omp parallel for + for (w = 0; w < numWorkers; w++) { uint8_t outCounter = 0; - TIFF* tif = TIFFOpen(fileName, "r"); - while(!tif){ + TIFF * tif = TIFFOpen(fileName, "r"); + while (!tif) { tif = TIFFOpen(fileName, "r"); - if(outCounter == 3){ - #pragma omp critical + if (outCounter == 3) { +#pragma omp critical { err = 1; - sprintf(errString,"Thread %d: File \"%s\" cannot be opened\n",w,fileName); + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); } continue; } outCounter++; } - void* buffer = malloc(x*stripSize*bytes); - for(int64_t dir = startSlice+(w*batchSize); dir < startSlice+((w+1)*batchSize); dir++){ - if(dir>=z+startSlice || err) break; + void *buffer = malloc(x * stripSize * bytes); + for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); + dir++) { + if (dir >= z + startSlice || err) + break; uint8_t counter = 0; - while(!TIFFSetDirectory(tif, (uint64_t)dir) && counter<3){ + while (!TIFFSetDirectory(tif, (uint64_t)dir) && counter < 3) { counter++; - if(counter == 3){ - #pragma omp critical + if (counter == 3) { +#pragma omp critical { err = 1; - sprintf(errString,"Thread %d: File \"%s\" cannot be opened\n",w,fileName); + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); } } } - if(err) break; - for (int64_t i = 0; i*stripSize < y; i++) - { + if (err) + break; + for (int64_t i = 0; i * stripSize < y; i++) { - //loading the data into a buffer - int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize*x*bytes); - if(cBytes < 0){ - #pragma omp critical + // loading the data into a buffer + int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize * x * bytes); + if (cBytes < 0) { +#pragma omp critical { errBak = 1; - err = 1; - sprintf(errString,"Thread %d: Strip %ld cannot be read\n",w,i); + err = 1; + sprintf(errString, "Thread %d: Strip %ld cannot be read\n", w, i); } break; } - if(!flipXY){ - memcpy(tiff+((i*stripSize*x)*bytes),buffer,cBytes); + if (!flipXY) { + memcpy(tiff + ((i * stripSize * x) * bytes), buffer, cBytes); continue; } - switch(bits){ + switch (bits) { case 8: // Map Values to flip x and y for MATLAB - for(int64_t k = 0; k < stripSize; k++){ - if((k+(i*stripSize)) >= y) break; - for(int64_t j = 0; j < x; j++){ - ((uint8_t*)tiff)[((j*y)+(k+(i*stripSize)))+((dir-startSlice)*(x*y))] = ((uint8_t*)buffer)[j+(k*x)]; + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((uint8_t *)buffer)[j + (k * x)]; } } - break; + break; case 16: // Map Values to flip x and y for MATLAB - for(int64_t k = 0; k < stripSize; k++){ - if((k+(i*stripSize)) >= y) break; - for(int64_t j = 0; j < x; j++){ - ((uint16_t*)tiff)[((j*y)+(k+(i*stripSize)))+((dir-startSlice)*(x*y))] = ((uint16_t*)buffer)[j+(k*x)]; + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((uint16_t *)buffer)[j + (k * x)]; } } - break; + break; case 32: // Map Values to flip x and y for MATLAB - for(int64_t k = 0; k < stripSize; k++){ - if((k+(i*stripSize)) >= y) break; - for(int64_t j = 0; j < x; j++){ - ((float*)tiff)[((j*y)+(k+(i*stripSize)))+((dir-startSlice)*(x*y))] = ((float*)buffer)[j+(k*x)]; + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((float *)buffer)[j + (k * x)]; } } - break; + break; case 64: // Map Values to flip x and y for MATLAB - for(int64_t k = 0; k < stripSize; k++){ - if((k+(i*stripSize)) >= y) break; - for(int64_t j = 0; j < x; j++){ - ((double*)tiff)[((j*y)+(k+(i*stripSize)))+((dir-startSlice)*(x*y))] = ((double*)buffer)[j+(k*x)]; + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((double *)buffer)[j + (k * x)]; } } - break; + break; } } } @@ -208,62 +207,68 @@ void readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char* fileName, TIFFClose(tif); } } - else{ - uint64_t stripsPerDir = (uint64_t)ceil((double)y/(double)stripSize); - #ifdef _WIN32 - int fd = open(fileName,O_RDONLY | O_BINARY); - #else - int fd = open(fileName,O_RDONLY); - #endif - if(fd == -1) mexErrMsgIdAndTxt("disk:threadError","File \"%s\" cannot be opened from Disk\n",fileName); - - if(!tif) mexErrMsgIdAndTxt("tiff:threadError","File \"%s\" cannot be opened\n",fileName); - uint64_t offset = 0; - uint64_t* offsets = NULL; + else { + uint64_t stripsPerDir = (uint64_t)ceil((double)y / (double)stripSize); +#ifdef _WIN32 + int fd = open(fileName, O_RDONLY | O_BINARY); +#else + int fd = open(fileName, O_RDONLY); +#endif + if (fd == -1) + printf("disk:threadError", "File \"%s\" cannot be opened from Disk\n", fileName); + + if (!tif) + printf("tiff:threadError", "File \"%s\" cannot be opened\n", fileName); + uint64_t offset = 0; + uint64_t *offsets = NULL; TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); - uint64_t* byteCounts = NULL; + uint64_t *byteCounts = NULL; TIFFGetField(tif, TIFFTAG_STRIPBYTECOUNTS, &byteCounts); - if(!offsets || !byteCounts) mexErrMsgIdAndTxt("tiff:threadError","Could not get offsets or byte counts from the tiff file\n"); - offset = offsets[0]; - uint64_t fOffset = offsets[stripsPerDir-1]+byteCounts[stripsPerDir-1]; - uint64_t zSize = fOffset-offset; - TIFFSetDirectory(tif,1); + if (!offsets || !byteCounts) + printf("tiff:threadError", "Could not get offsets or byte counts from the tiff file\n"); + offset = offsets[0]; + uint64_t fOffset = offsets[stripsPerDir - 1] + byteCounts[stripsPerDir - 1]; + uint64_t zSize = fOffset - offset; + TIFFSetDirectory(tif, 1); TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); - uint64_t gap = offsets[0]-fOffset; - - lseek(fd, offset, SEEK_SET); + uint64_t gap = offsets[0] - fOffset; + lseek(fd, offset, SEEK_SET); TIFFClose(tif); - uint64_t curr = 0; + uint64_t curr = 0; uint64_t bytesRead = 0; // TESTING // Not sure if we will need to read in chunks like for ImageJ - for(uint64_t i = 0; i < z; i++){ - bytesRead = read(fd,tiff+curr,zSize); + for (uint64_t i = 0; i < z; i++) { + bytesRead = read(fd, tiff + curr, zSize); curr += bytesRead; - lseek(fd,gap,SEEK_CUR); + lseek(fd, gap, SEEK_CUR); } close(fd); - uint64_t size = x*y*z*(bits/8); - void* tiffC = malloc(size); - memcpy(tiffC,tiff,size); - #pragma omp parallel for - for(uint64_t k = 0; k < z; k++){ - for(uint64_t j = 0; j < x; j++){ - for(uint64_t i = 0; i < y; i++){ - switch(bits){ + uint64_t size = x * y * z * (bits / 8); + void * tiffC = malloc(size); + memcpy(tiffC, tiff, size); +#pragma omp parallel for + for (uint64_t k = 0; k < z; k++) { + for (uint64_t j = 0; j < x; j++) { + for (uint64_t i = 0; i < y; i++) { + switch (bits) { case 8: - ((uint8_t*)tiff)[i+(j*y)+(k*x*y)] = ((uint8_t*)tiffC)[j+(i*x)+(k*x*y)]; + ((uint8_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint8_t *)tiffC)[j + (i * x) + (k * x * y)]; break; case 16: - ((uint16_t*)tiff)[i+(j*y)+(k*x*y)] = ((uint16_t*)tiffC)[j+(i*x)+(k*x*y)]; + ((uint16_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint16_t *)tiffC)[j + (i * x) + (k * x * y)]; break; case 32: - ((float*)tiff)[i+(j*y)+(k*x*y)] = ((float*)tiffC)[j+(i*x)+(k*x*y)]; + ((float *)tiff)[i + (j * y) + (k * x * y)] = + ((float *)tiffC)[j + (i * x) + (k * x * y)]; break; case 64: - ((double*)tiff)[i+(j*y)+(k*x*y)] = ((double*)tiffC)[j+(i*x)+(k*x*y)]; + ((double *)tiff)[i + (j * y) + (k * x * y)] = + ((double *)tiffC)[j + (i * x) + (k * x * y)]; break; } } @@ -271,69 +276,81 @@ void readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char* fileName, } free(tiffC); } - if(err){ - if(errBak) readTiffParallelBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); - else mexErrMsgIdAndTxt("tiff:threadError",errString); + if (err) { + if (errBak) + readTiffParallelBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); + else + printf("tiff:threadError", errString); } } // Backup method in case there are errors reading strips -void readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char* fileName, void* tiff, uint64_t bits, uint64_t startSlice, uint8_t flipXY){ - int32_t numWorkers = omp_get_max_threads(); - int32_t batchSize = (y-1)/numWorkers+1; - uint64_t bytes = bits/8; +void +readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (y - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; int32_t w; - #pragma omp parallel for - for(w = 0; w < numWorkers; w++){ +#pragma omp parallel for + for (w = 0; w < numWorkers; w++) { - TIFF* tif = TIFFOpen(fileName, "r"); - if(!tif) mexErrMsgIdAndTxt("tiff:threadError","Thread %d: File \"%s\" cannot be opened\n",w,fileName); + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:threadError", "Thread %d: File \"%s\" cannot be opened\n", w, fileName); - void* buffer = malloc(x*bytes); - for(int64_t dir = startSlice+(w*batchSize); dir < startSlice+((w+1)*batchSize); dir++){ - if(dir>=z+startSlice) break; + void *buffer = malloc(x * bytes); + for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); dir++) { + if (dir >= z + startSlice) + break; int counter = 0; - while(!TIFFSetDirectory(tif, (uint64_t)0) && counter<3){ - printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n",w,fileName,dir,counter+1); + while (!TIFFSetDirectory(tif, (uint64_t)0) && counter < 3) { + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, dir, + counter + 1); counter++; } - for (int64_t i = (w*batchSize); i < ((w+1)*batchSize); i++) - { - if(i >= y) break; + for (int64_t i = (w * batchSize); i < ((w + 1) * batchSize); i++) { + if (i >= y) + break; TIFFReadScanline(tif, buffer, i, 0); - if(!flipXY){ - memcpy(tiff+((i*x)*bytes),buffer,x*bytes); + if (!flipXY) { + memcpy(tiff + ((i * x) * bytes), buffer, x * bytes); continue; } - //loading the data into a buffer - switch(bits){ + // loading the data into a buffer + switch (bits) { case 8: // Map Values to flip x and y for MATLAB - for(int64_t j = 0; j < x; j++){ - ((uint8_t*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((uint8_t*)buffer)[j]; + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint8_t *)buffer)[j]; } - break; + break; case 16: // Map Values to flip x and y for MATLAB - for(int64_t j = 0; j < x; j++){ - ((uint16_t*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((uint16_t*)buffer)[j]; + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint16_t *)buffer)[j]; } - break; + break; case 32: // Map Values to flip x and y for MATLAB - for(int64_t j = 0; j < x; j++){ - ((float*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((float*)buffer)[j]; + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((float *)buffer)[j]; } - break; + break; case 64: // Map Values to flip x and y for MATLAB - for(int64_t j = 0; j < x; j++){ - ((double*)tiff)[((j*y)+i)+((dir-startSlice)*(x*y))] = ((double*)buffer)[j]; + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((double *)buffer)[j]; } - break; + break; } } } @@ -342,135 +359,152 @@ void readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char* fileN } } -void readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char* fileName, void* tiff, uint64_t bits, uint64_t startSlice, uint64_t stripSize, uint8_t flipXY){ - int32_t numWorkers = omp_get_max_threads(); - uint64_t stripsPerDir = (uint64_t)ceil((double)y/(double)stripSize); - int32_t batchSize = (stripsPerDir-1)/numWorkers+1; - uint64_t bytes = bits/8; +void +readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + uint64_t stripsPerDir = (uint64_t)ceil((double)y / (double)stripSize); + int32_t batchSize = (stripsPerDir - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; int32_t w; - uint8_t err = 0; + uint8_t err = 0; uint8_t errBak = 0; - char errString[10000]; - + char errString[10000]; - #pragma omp parallel for - for(w = 0; w < numWorkers; w++){ +#pragma omp parallel for + for (w = 0; w < numWorkers; w++) { uint8_t outCounter = 0; - TIFF* tif = TIFFOpen(fileName, "r"); - while(!tif){ + TIFF * tif = TIFFOpen(fileName, "r"); + while (!tif) { tif = TIFFOpen(fileName, "r"); - if(outCounter == 3){ - #pragma omp critical + if (outCounter == 3) { +#pragma omp critical { err = 1; - sprintf(errString,"Thread %d: File \"%s\" cannot be opened\n",w,fileName); + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); } continue; } outCounter++; } - void* buffer = malloc(x*stripSize*bytes); - + void *buffer = malloc(x * stripSize * bytes); uint8_t counter = 0; - while(!TIFFSetDirectory(tif, 0) && counter<3){ - printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n",w,fileName,0,counter+1); + while (!TIFFSetDirectory(tif, 0) && counter < 3) { + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, 0, + counter + 1); counter++; - if(counter == 3){ - #pragma omp critical + if (counter == 3) { +#pragma omp critical { err = 1; - sprintf(errString,"Thread %d: File \"%s\" cannot be opened\n",w,fileName); + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); } } } - for (int64_t i = (w*batchSize); i < (w+1)*batchSize; i++) - { - if(i*stripSize >= y || err) break; - //loading the data into a buffer - int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize*x*bytes); - if(cBytes < 0){ - #pragma omp critical + for (int64_t i = (w * batchSize); i < (w + 1) * batchSize; i++) { + if (i * stripSize >= y || err) + break; + // loading the data into a buffer + int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize * x * bytes); + if (cBytes < 0) { +#pragma omp critical { errBak = 1; - err = 1; - sprintf(errString,"Thread %d: Strip %ld cannot be read\n",w,i); + err = 1; + sprintf(errString, "Thread %d: Strip %ld cannot be read\n", w, i); } break; } - if(!flipXY){ - memcpy(tiff+((i*stripSize*x)*bytes),buffer,cBytes); + if (!flipXY) { + memcpy(tiff + ((i * stripSize * x) * bytes), buffer, cBytes); continue; } - switch(bits){ + switch (bits) { case 8: // Map Values to flip x and y for MATLAB - for(int64_t k = 0; k < stripSize; k++){ - if((k+(i*stripSize)) >= y) break; - for(int64_t j = 0; j < x; j++){ - ((uint8_t*)tiff)[((j*y)+(k+(i*stripSize)))] = ((uint8_t*)buffer)[j+(k*x)]; + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((uint8_t *)buffer)[j + (k * x)]; } } - break; + break; case 16: // Map Values to flip x and y for MATLAB - for(int64_t k = 0; k < stripSize; k++){ - if((k+(i*stripSize)) >= y) break; - for(int64_t j = 0; j < x; j++){ - ((uint16_t*)tiff)[((j*y)+(k+(i*stripSize)))] = ((uint16_t*)buffer)[j+(k*x)]; + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((uint16_t *)buffer)[j + (k * x)]; } } - break; + break; case 32: // Map Values to flip x and y for MATLAB - for(int64_t k = 0; k < stripSize; k++){ - if((k+(i*stripSize)) >= y) break; - for(int64_t j = 0; j < x; j++){ - ((float*)tiff)[((j*y)+(k+(i*stripSize)))] = ((float*)buffer)[j+(k*x)]; + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((float *)buffer)[j + (k * x)]; } } - break; + break; case 64: // Map Values to flip x and y for MATLAB - for(int64_t k = 0; k < stripSize; k++){ - if((k+(i*stripSize)) >= y) break; - for(int64_t j = 0; j < x; j++){ - ((double*)tiff)[((j*y)+(k+(i*stripSize)))] = ((double*)buffer)[j+(k*x)]; + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((double *)buffer)[j + (k * x)]; } } - break; + break; } } free(buffer); TIFFClose(tif); } - if(err) { - if(errBak) readTiffParallel2DBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); - else mexErrMsgIdAndTxt("tiff:threadError",errString); + if (err) { + if (errBak) + readTiffParallel2DBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); + else + printf("tiff:threadError", errString); } } // Reading images saved by ImageJ -void readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char* fileName, void* tiff, uint64_t bits, uint64_t startSlice, uint64_t stripSize, uint8_t flipXY){ - #ifdef _WIN32 - int fd = open(fileName,O_RDONLY | O_BINARY); - #else - int fd = open(fileName,O_RDONLY); - #endif - TIFF* tif = TIFFOpen(fileName, "r"); - if(!tif) mexErrMsgIdAndTxt("tiff:threadError","File \"%s\" cannot be opened\n",fileName); - uint64_t offset = 0; - uint64_t* offsets = NULL; +void +readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY) +{ +#ifdef _WIN32 + int fd = open(fileName, O_RDONLY | O_BINARY); +#else + int fd = open(fileName, O_RDONLY); +#endif + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:threadError", "File \"%s\" cannot be opened\n", fileName); + uint64_t offset = 0; + uint64_t *offsets = NULL; TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); - if(offsets) offset = offsets[0]; + if (offsets) + offset = offsets[0]; TIFFClose(tif); lseek(fd, offset, SEEK_SET); - uint64_t bytes = bits/8; + uint64_t bytes = bits / 8; //#pragma omp parallel for /* for(uint64_t i = 0; i < z; i++){ @@ -478,75 +512,84 @@ void readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char* file //pread(fd,tiff+cOffset,x*y*bytes,offset+cOffset); read(fd,tiff+cOffset,x*y*bytes); }*/ - uint64_t chunk = 0; - uint64_t tBytes = x*y*z*bytes; + uint64_t chunk = 0; + uint64_t tBytes = x * y * z * bytes; uint64_t bytesRead; uint64_t rBytes = tBytes; - if(tBytes < INT_MAX) bytesRead = read(fd,tiff,tBytes); - else{ - while(chunk < tBytes){ - rBytes = tBytes-chunk; - if(rBytes > INT_MAX) bytesRead = read(fd,tiff+chunk,INT_MAX); - else bytesRead = read(fd,tiff+chunk,rBytes); + if (tBytes < INT_MAX) + bytesRead = read(fd, tiff, tBytes); + else { + while (chunk < tBytes) { + rBytes = tBytes - chunk; + if (rBytes > INT_MAX) + bytesRead = read(fd, tiff + chunk, INT_MAX); + else + bytesRead = read(fd, tiff + chunk, rBytes); chunk += bytesRead; } } close(fd); // Swap endianess for types greater than 8 bits // TODO: May need to change later because we may not always need to swap - if(bits > 8){ - #pragma omp parallel for - for(uint64_t i = 0; i < x*y*z; i++){ - switch(bits){ + if (bits > 8) { +#pragma omp parallel for + for (uint64_t i = 0; i < x * y * z; i++) { + switch (bits) { case 16: - //((uint16_t*)tiff)[i] = ((((uint16_t*)tiff)[i] & 0xff) >> 8) | (((uint16_t*)tiff)[i] << 8); + //((uint16_t*)tiff)[i] = ((((uint16_t*)tiff)[i] & 0xff) >> 8) | (((uint16_t*)tiff)[i] << + // 8); //((uint16_t*)tiff)[i] = bswap_16(((uint16_t*)tiff)[i]); - ((uint16_t*)tiff)[i] = ((((uint16_t*)tiff)[i] << 8) & 0xff00) | ((((uint16_t*)tiff)[i] >> 8) & 0x00ff); + ((uint16_t *)tiff)[i] = + ((((uint16_t *)tiff)[i] << 8) & 0xff00) | ((((uint16_t *)tiff)[i] >> 8) & 0x00ff); break; case 32: - //((num & 0xff000000) >> 24) | ((num & 0x00ff0000) >> 8) | ((num & 0x0000ff00) << 8) | (num << 24) + //((num & 0xff000000) >> 24) | ((num & 0x00ff0000) >> 8) | ((num & 0x0000ff00) << 8) | + //(num << 24) //((float*)tiff)[i] = bswap_32(((float*)tiff)[i]); - ((uint32_t*)tiff)[i] = ((((uint32_t*)tiff)[i] << 24) & 0xff000000 ) | - ((((uint32_t*)tiff)[i] << 8) & 0x00ff0000 ) | - ((((uint32_t*)tiff)[i] >> 8) & 0x0000ff00 ) | - ((((uint32_t*)tiff)[i] >> 24) & 0x000000ff ); + ((uint32_t *)tiff)[i] = ((((uint32_t *)tiff)[i] << 24) & 0xff000000) | + ((((uint32_t *)tiff)[i] << 8) & 0x00ff0000) | + ((((uint32_t *)tiff)[i] >> 8) & 0x0000ff00) | + ((((uint32_t *)tiff)[i] >> 24) & 0x000000ff); break; case 64: //((double*)tiff)[i] = bswap_64(((double*)tiff)[i]); - ((uint64_t*)tiff)[i] = ( (((uint64_t*)tiff)[i] << 56) & 0xff00000000000000UL ) | - ( (((uint64_t*)tiff)[i] << 40) & 0x00ff000000000000UL ) | - ( (((uint64_t*)tiff)[i] << 24) & 0x0000ff0000000000UL ) | - ( (((uint64_t*)tiff)[i] << 8) & 0x000000ff00000000UL ) | - ( (((uint64_t*)tiff)[i] >> 8) & 0x00000000ff000000UL ) | - ( (((uint64_t*)tiff)[i] >> 24) & 0x0000000000ff0000UL ) | - ( (((uint64_t*)tiff)[i] >> 40) & 0x000000000000ff00UL ) | - ( (((uint64_t*)tiff)[i] >> 56) & 0x00000000000000ffUL ); + ((uint64_t *)tiff)[i] = ((((uint64_t *)tiff)[i] << 56) & 0xff00000000000000UL) | + ((((uint64_t *)tiff)[i] << 40) & 0x00ff000000000000UL) | + ((((uint64_t *)tiff)[i] << 24) & 0x0000ff0000000000UL) | + ((((uint64_t *)tiff)[i] << 8) & 0x000000ff00000000UL) | + ((((uint64_t *)tiff)[i] >> 8) & 0x00000000ff000000UL) | + ((((uint64_t *)tiff)[i] >> 24) & 0x0000000000ff0000UL) | + ((((uint64_t *)tiff)[i] >> 40) & 0x000000000000ff00UL) | + ((((uint64_t *)tiff)[i] >> 56) & 0x00000000000000ffUL); break; } - } } // Find a way to do this in-place without making a copy - if(flipXY){ - uint64_t size = x*y*z*(bits/8); - void* tiffC = malloc(size); - memcpy(tiffC,tiff,size); - #pragma omp parallel for - for(uint64_t k = 0; k < z; k++){ - for(uint64_t j = 0; j < x; j++){ - for(uint64_t i = 0; i < y; i++){ - switch(bits){ + if (flipXY) { + uint64_t size = x * y * z * (bits / 8); + void * tiffC = malloc(size); + memcpy(tiffC, tiff, size); +#pragma omp parallel for + for (uint64_t k = 0; k < z; k++) { + for (uint64_t j = 0; j < x; j++) { + for (uint64_t i = 0; i < y; i++) { + switch (bits) { case 8: - ((uint8_t*)tiff)[i+(j*y)+(k*x*y)] = ((uint8_t*)tiffC)[j+(i*x)+(k*x*y)]; + ((uint8_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint8_t *)tiffC)[j + (i * x) + (k * x * y)]; break; case 16: - ((uint16_t*)tiff)[i+(j*y)+(k*x*y)] = ((uint16_t*)tiffC)[j+(i*x)+(k*x*y)]; + ((uint16_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint16_t *)tiffC)[j + (i * x) + (k * x * y)]; break; case 32: - ((float*)tiff)[i+(j*y)+(k*x*y)] = ((float*)tiffC)[j+(i*x)+(k*x*y)]; + ((float *)tiff)[i + (j * y) + (k * x * y)] = + ((float *)tiffC)[j + (i * x) + (k * x * y)]; break; case 64: - ((double*)tiff)[i+(j*y)+(k*x*y)] = ((double*)tiffC)[j+(i*x)+(k*x*y)]; + ((double *)tiff)[i + (j * y) + (k * x * y)] = + ((double *)tiffC)[j + (i * x) + (k * x * y)]; break; } } @@ -556,207 +599,390 @@ void readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char* file } } -uint8_t isImageJIm(const char* fileName){ - TIFF* tif = TIFFOpen(fileName, "r"); - if(!tif) return 0; - char* tiffDesc = NULL; - if(TIFFGetField(tif, TIFFTAG_IMAGEDESCRIPTION, &tiffDesc)){ - if(strstr(tiffDesc, "ImageJ")){ +uint8_t +isImageJIm(TIFF *tif) +{ + if (!tif) + return 0; + char *tiffDesc = NULL; + if (TIFFGetField(tif, TIFFTAG_IMAGEDESCRIPTION, &tiffDesc)) { + if (strstr(tiffDesc, "ImageJ")) { return 1; } } return 0; } -uint64_t imageJImGetZ(const char* fileName){ - TIFF* tif = TIFFOpen(fileName, "r"); - if(!tif) return 0; - char* tiffDesc = NULL; - if(TIFFGetField(tif, TIFFTAG_IMAGEDESCRIPTION, &tiffDesc)){ - if(strstr(tiffDesc, "ImageJ")){ - char* nZ = strstr(tiffDesc,"images="); - if(nZ){ - nZ+=7; - char* temp; - return strtol(nZ,&temp,10); +uint64_t +imageJImGetZ(TIFF *tif) +{ + if (!tif) + return 0; + char *tiffDesc = NULL; + if (TIFFGetField(tif, TIFFTAG_IMAGEDESCRIPTION, &tiffDesc)) { + if (strstr(tiffDesc, "ImageJ")) { + char *nZ = strstr(tiffDesc, "images="); + if (nZ) { + nZ += 7; + char *temp; + return strtol(nZ, &temp, 10); } } } return 0; } -void mexFunction(int nlhs, mxArray *plhs[], - int nrhs, const mxArray *prhs[]) +void +get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, uint64_t *y, uint64_t *z, + uint64_t *bits, uint64_t *startSlice, uint64_t *stripSize, uint64_t *is_imageJ, + uint64_t *imageJ_Z) { - // Check if the fileName is a char array or matlab style - char* fileName = NULL; - if(!mxIsClass(prhs[0], "string")){ - if(!mxIsChar(prhs[0])) mexErrMsgIdAndTxt("tiff:inputError","The first argument must be a string"); - fileName = mxArrayToString(prhs[0]); - } - else{ - mxArray* mString[1]; - mxArray* mCharA[1]; - - // Convert string to char array - mString[0] = mxDuplicateArray(prhs[0]); - mexCallMATLAB(1, mCharA, 1, mString, "char"); - fileName = mxArrayToString(mCharA[0]); - } - - // Handle the tilde character in filenames on Linux/Mac - #ifndef _WIN32 - if(strchr(fileName,'~')) fileName = expandTilde(fileName); - #endif - - uint8_t flipXY = 1; - //uint8_t flipXY = 0; - - - //if(nrhs > 2){ - // flipXY = (uint8_t)*(mxGetPr(prhs[2])); - //} - - TIFFSetWarningHandler(DummyHandler); - TIFF* tif = TIFFOpen(fileName, "r"); - if(!tif) mexErrMsgIdAndTxt("tiff:inputError","File \"%s\" cannot be opened",fileName); + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:inputError", "File \"%s\" cannot be opened", fileName); - uint64_t x = 1,y = 1,z = 1,bits = 1, startSlice = 0; - TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, &x); - TIFFGetField(tif, TIFFTAG_IMAGELENGTH, &y); + TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, x); + TIFFGetField(tif, TIFFTAG_IMAGELENGTH, y); - if(nrhs == 1){ + if (strip_range == NULL) { uint16_t s = 0, m = 0, t = 1; - while(TIFFSetDirectory(tif,t)){ + while (TIFFSetDirectory(tif, t)) { s = t; t *= 8; - if(s > t){ + if (s > t) { t = 65535; printf("Number of slices > 32768\n"); break; } } - while(s != t){ - m = (s+t+1)/2; - if(TIFFSetDirectory(tif,m)){ + while (s != t) { + m = (s + t + 1) / 2; + if (TIFFSetDirectory(tif, m)) { s = m; } - else{ - if(m > 0) t = m-1; - else t = m; + else { + if (m > 0) + t = m - 1; + else + t = m; } } - z = s+1; + z = s + 1; } - else{ - if(mxGetN(prhs[1]) != 2){ - mexErrMsgIdAndTxt("tiff:inputError","Input range is not 2"); - } - else{ - startSlice = (uint64_t)*(mxGetPr(prhs[1]))-1; - z = (uint64_t)*((mxGetPr(prhs[1])+1))-startSlice; - if (!TIFFSetDirectory(tif,startSlice+z-1) || !TIFFSetDirectory(tif,startSlice)){ - mexErrMsgIdAndTxt("tiff:rangeOutOfBound","Range is out of bounds"); + else { + if (strip_range->length != 2) { + printf("tiff:inputError", "Input range is not 2"); + } + else { + *startSlice = (uint64_t) * (strip_range->range) - 1; + *z = (uint64_t) * (strip_range->range + 1)) - startSlice; + if (!TIFFSetDirectory(tif, startSlice[0] + z[0] - 1) || !TIFFSetDirectory(tif, startSlice[0])) { + printf("tiff:rangeOutOfBound", "Range is out of bounds"); } } } - TIFFGetField(tif, TIFFTAG_BITSPERSAMPLE, &bits); - uint64_t stripSize = 1; - TIFFGetField(tif, TIFFTAG_ROWSPERSTRIP, &stripSize); + *is_imageJ = isImageJIm(tif); + *imageJ_Z = imageJImGetZ(tif); + if (*is_imageJ) { + *is_imageJ = 1; + *imageJ_Z = imageJImGetZ(fileName); + if (*imageJ_Z) + *z = *imageJ_Z; + } + + TIFFGetField(tif, TIFFTAG_BITSPERSAMPLE, bits); + TIFFGetField(tif, TIFFTAG_ROWSPERSTRIP, stripSize); TIFFClose(tif); +} + +uint8_t *** +createU8Array(int ndim, size_t *dim) +{ + size_t i, j; + uint8_t ***array = (uint8_t ***)malloc(dim[0] * sizeof(uint8_t **)); + for (i = 0; i < dim[0]; i++) { + array[i] = (uint8_t **)malloc(dim[1] * sizeof(uint8_t *)); + for (j = 0; j < dim[1]; j++) { + array[i][j] = (uint8_t *)calloc(dim[2], sizeof(uint8_t)); + } + } + return array; +} - uint8_t imageJIm = 0; - if(isImageJIm(fileName)){ - imageJIm = 1; - uint64_t tempZ = imageJImGetZ(fileName); - if(tempZ) z = tempZ; +uint16_t *** +createU16Array(int ndim, size_t *dim) +{ + size_t i, j; + uint16_t ***array = (uint16_t ***)malloc(dim[0] * sizeof(uint16_t **)); + for (i = 0; i < dim[0]; i++) { + array[i] = (uint16_t **)malloc(dim[1] * sizeof(uint16_t *)); + for (j = 0; j < dim[1]; j++) { + array[i][j] = (uint16_t *)calloc(dim[2], sizeof(uint16_t)); + } } + return array; +} - uint64_t dim[3]; - dim[0] = y; - dim[1] = x; - dim[2] = z; +float *** +createFloatArray(int ndim, size_t *dim) +{ + size_t i, j; + float ***array = (float ***)malloc(dim[0] * sizeof(float **)); + for (i = 0; i < dim[0]; i++) { + array[i] = (float **)malloc(dim[1] * sizeof(float *)); + for (j = 0; j < dim[1]; j++) { + array[i][j] = (float *)calloc(dim[2], sizeof(float)); + } + } + return array; +} +double *** +createDoubleArray(int ndim, size_t *dim) +{ + size_t i, j; + double ***array = (double ***)malloc(dim[0] * sizeof(double **)); + for (i = 0; i < dim[0]; i++) { + array[i] = (double **)malloc(dim[1] * sizeof(double *)); + for (j = 0; j < dim[1]; j++) { + array[i][j] = (double *)calloc(dim[2], sizeof(double)); + } + } + return array; +} +void +_get_tiff_array(int bits, int ndim, size_t *dim) +{ + void *tiff = NULL; + if (bits == 8) { + tiff = (void *)createU8Array(ndim, dims); + } + else if (bits == 16) { + tiff = (void *)createU16Array(ndim, dims); + } + else if (bits == 32) { + tiff = (void *)createFloatArray(ndim, dims); + } + else if (bits == 64) { + tiff = (void *)createDoubleArray(ndim, dims); + } + return tiff; +} +void +_TIFF_load(char *fileName, uint64_t x, uint64_t y, uint64_t z, uint64_t bits, uint64_t startSlice, + uint64_t stripSize, uint8_t flipXY, int ndim, sizt_t *dims, void **tiff_ptr) +{ + if (tiff == NULL) { + printf("tiff:dataTypeError, Data type not suppported\n"); + } + *tiff_ptr = _get_tiff_array(bits, ndim, dims); // Case for ImageJ - if(imageJIm){ - if(bits == 8){ - plhs[0] = mxCreateNumericArray(3,dim,mxUINT8_CLASS, mxREAL); - uint8_t* tiff = (uint8_t*)mxGetPr(plhs[0]); - readTiffParallelImageJ(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else if(bits == 16){ - plhs[0] = mxCreateNumericArray(3,dim,mxUINT16_CLASS, mxREAL); - uint16_t* tiff = (uint16_t*)mxGetPr(plhs[0]); - readTiffParallelImageJ(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else if(bits == 32){ - plhs[0] = mxCreateNumericArray(3,dim,mxSINGLE_CLASS, mxREAL); - float* tiff = (float*)mxGetPr(plhs[0]); - readTiffParallelImageJ(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else if(bits == 64){ - plhs[0] = mxCreateNumericArray(3,dim,mxDOUBLE_CLASS, mxREAL); - double* tiff = (double*)mxGetPr(plhs[0]); - readTiffParallelImageJ(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else{ - mexErrMsgIdAndTxt("tiff:dataTypeError","Data type not suppported"); - } + if (imageJIm) { + readTiffParallelImageJ(x, y, z, fileName, *tiff_ptr, bits, startSlice, stripSize, flipXY); } // Case for 2D - else if(z <= 1){ - if(bits == 8){ - plhs[0] = mxCreateNumericArray(3,dim,mxUINT8_CLASS, mxREAL); - uint8_t* tiff = (uint8_t*)mxGetPr(plhs[0]); - readTiffParallel2D(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else if(bits == 16){ - plhs[0] = mxCreateNumericArray(3,dim,mxUINT16_CLASS, mxREAL); - uint16_t* tiff = (uint16_t*)mxGetPr(plhs[0]); - readTiffParallel2D(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else if(bits == 32){ - plhs[0] = mxCreateNumericArray(3,dim,mxSINGLE_CLASS, mxREAL); - float* tiff = (float*)mxGetPr(plhs[0]); - readTiffParallel2D(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else if(bits == 64){ - plhs[0] = mxCreateNumericArray(3,dim,mxDOUBLE_CLASS, mxREAL); - double* tiff = (double*)mxGetPr(plhs[0]); - readTiffParallel2D(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else{ - mexErrMsgIdAndTxt("tiff:dataTypeError","Data type not suppported"); - } + else if (z <= 1) { + readTiffParallel2D(x, y, z, fileName, *tiff_ptr, bits, startSlice, stripSize, flipXY); } // Case for 3D - else{ - if(bits == 8){ - plhs[0] = mxCreateNumericArray(3,dim,mxUINT8_CLASS, mxREAL); - uint8_t* tiff = (uint8_t*)mxGetPr(plhs[0]); - readTiffParallel(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else if(bits == 16){ - plhs[0] = mxCreateNumericArray(3,dim,mxUINT16_CLASS, mxREAL); - uint16_t* tiff = (uint16_t*)mxGetPr(plhs[0]); - readTiffParallel(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else if(bits == 32){ - plhs[0] = mxCreateNumericArray(3,dim,mxSINGLE_CLASS, mxREAL); - float* tiff = (float*)mxGetPr(plhs[0]); - readTiffParallel(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else if(bits == 64){ - plhs[0] = mxCreateNumericArray(3,dim,mxDOUBLE_CLASS, mxREAL); - double* tiff = (double*)mxGetPr(plhs[0]); - readTiffParallel(x,y,z,fileName, (void*)tiff, bits, startSlice, stripSize, flipXY); - } - else{ - mexErrMsgIdAndTxt("tiff:dataTypeError","Data type not suppported"); - } + else { + readTiffParallel(x, y, z, fileName, *tiff_ptr, bits, startSlice, stripSize, flipXY); } -} \ No newline at end of file +} + +void +parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tiff_range_t *strip_range) +{ + uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 0, is_imageJ = 0, imageJ_Z = 0; + + get_tif_info(file_name, strip_range, &x, &y, &z, &bits, &startSlice, &stripeSize, &is_imageJ, &imageJ_Z); + + int ndim = 3; + uint64_t dims[ndim]; + dims[0] = flipXY ? y : x; + dims[1] = flipXY ? x : y; + dims[2] = z; + + _TIFF_load(fileName, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, tiff_ptr); +} + +// void +// mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) +// { +// // Check if the fileName is a char array or matlab style +// char *fileName = NULL; +// if (!mxIsClass(prhs[0], "string")) { +// if (!mxIsChar(prhs[0])) +// printf("tiff:inputError", "The first argument must be a string"); +// fileName = mxArrayToString(prhs[0]); +// } +// else { +// mxArray *mString[1]; +// mxArray *mCharA[1]; + +// // Convert string to char array +// mString[0] = mxDuplicateArray(prhs[0]); +// mexCallMATLAB(1, mCharA, 1, mString, "char"); +// fileName = mxArrayToString(mCharA[0]); +// } + +// // Handle the tilde character in filenames on Linux/Mac +// // #ifndef _WIN32 +// // if(strchr(fileName,'~')) fileName = expandTilde(fileName); +// // #endif + +// uint8_t flipXY = 1; +// // uint8_t flipXY = 0; + +// // if(nrhs > 2){ +// // flipXY = (uint8_t)*(mxGetPr(prhs[2])); +// //} + +// TIFFSetWarningHandler(DummyHandler); +// TIFF *tif = TIFFOpen(fileName, "r"); +// if (!tif) +// printf("tiff:inputError", "File \"%s\" cannot be opened", fileName); + +// uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0; +// TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, &x); +// TIFFGetField(tif, TIFFTAG_IMAGELENGTH, &y); + +// if (nrhs == 1) { +// uint16_t s = 0, m = 0, t = 1; +// while (TIFFSetDirectory(tif, t)) { +// s = t; +// t *= 8; +// if (s > t) { +// t = 65535; +// printf("Number of slices > 32768\n"); +// break; +// } +// } +// while (s != t) { +// m = (s + t + 1) / 2; +// if (TIFFSetDirectory(tif, m)) { +// s = m; +// } +// else { +// if (m > 0) +// t = m - 1; +// else +// t = m; +// } +// } +// z = s + 1; +// } +// else { +// if (mxGetN(prhs[1]) != 2) { +// printf("tiff:inputError", "Input range is not 2"); +// } +// else { +// startSlice = (uint64_t) * (mxGetPr(prhs[1])) - 1; +// z = (uint64_t) * ((mxGetPr(prhs[1]) + 1)) - startSlice; +// if (!TIFFSetDirectory(tif, startSlice + z - 1) || !TIFFSetDirectory(tif, startSlice)) { +// printf("tiff:rangeOutOfBound", "Range is out of bounds"); +// } +// } +// } + +// TIFFGetField(tif, TIFFTAG_BITSPERSAMPLE, &bits); +// uint64_t stripSize = 1; +// TIFFGetField(tif, TIFFTAG_ROWSPERSTRIP, &stripSize); +// TIFFClose(tif); + +// uint8_t imageJIm = 0; +// if (isImageJIm(fileName)) { +// imageJIm = 1; +// uint64_t tempZ = imageJImGetZ(fileName); +// if (tempZ) +// z = tempZ; +// } + +// uint64_t dim[3]; +// dim[0] = y; +// dim[1] = x; +// dim[2] = z; + +// // Case for ImageJ +// if (imageJIm) { +// if (bits == 8) { +// plhs[0] = mxCreateNumericArray(3, dim, mxUINT8_CLASS, mxREAL); +// uint8_t *tiff = (uint8_t *)mxGetPr(plhs[0]); +// readTiffParallelImageJ(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else if (bits == 16) { +// plhs[0] = mxCreateNumericArray(3, dim, mxUINT16_CLASS, mxREAL); +// uint16_t *tiff = (uint16_t *)mxGetPr(plhs[0]); +// readTiffParallelImageJ(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else if (bits == 32) { +// plhs[0] = mxCreateNumericArray(3, dim, mxSINGLE_CLASS, mxREAL); +// float *tiff = (float *)mxGetPr(plhs[0]); +// readTiffParallelImageJ(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else if (bits == 64) { +// plhs[0] = mxCreateNumericArray(3, dim, mxDOUBLE_CLASS, mxREAL); +// double *tiff = (double *)mxGetPr(plhs[0]); +// readTiffParallelImageJ(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else { +// printf("tiff:dataTypeError", "Data type not suppported"); +// } +// } +// // Case for 2D +// else if (z <= 1) { +// if (bits == 8) { +// plhs[0] = mxCreateNumericArray(3, dim, mxUINT8_CLASS, mxREAL); +// uint8_t *tiff = (uint8_t *)mxGetPr(plhs[0]); +// readTiffParallel2D(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else if (bits == 16) { +// plhs[0] = mxCreateNumericArray(3, dim, mxUINT16_CLASS, mxREAL); +// uint16_t *tiff = (uint16_t *)mxGetPr(plhs[0]); +// readTiffParallel2D(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else if (bits == 32) { +// plhs[0] = mxCreateNumericArray(3, dim, mxSINGLE_CLASS, mxREAL); +// float *tiff = (float *)mxGetPr(plhs[0]); +// readTiffParallel2D(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else if (bits == 64) { +// plhs[0] = mxCreateNumericArray(3, dim, mxDOUBLE_CLASS, mxREAL); +// double *tiff = (double *)mxGetPr(plhs[0]); +// readTiffParallel2D(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else { +// printf("tiff:dataTypeError", "Data type not suppported"); +// } +// } +// // Case for 3D +// else { +// if (bits == 8) { +// plhs[0] = mxCreateNumericArray(3, dim, mxUINT8_CLASS, mxREAL); +// uint8_t *tiff = (uint8_t *)mxGetPr(plhs[0]); +// readTiffParallel(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else if (bits == 16) { +// plhs[0] = mxCreateNumericArray(3, dim, mxUINT16_CLASS, mxREAL); +// uint16_t *tiff = (uint16_t *)mxGetPr(plhs[0]); +// readTiffParallel(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else if (bits == 32) { +// plhs[0] = mxCreateNumericArray(3, dim, mxSINGLE_CLASS, mxREAL); +// float *tiff = (float *)mxGetPr(plhs[0]); +// readTiffParallel(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else if (bits == 64) { +// plhs[0] = mxCreateNumericArray(3, dim, mxDOUBLE_CLASS, mxREAL); +// double *tiff = (double *)mxGetPr(plhs[0]); +// readTiffParallel(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); +// } +// else { +// printf("tiff:dataTypeError", "Data type not suppported"); +// } +// } +// } \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h index e69de29bb..5745216e0 100644 --- a/tools/llsm/parallelReadTiff.h +++ b/tools/llsm/parallelReadTiff.h @@ -0,0 +1,22 @@ +#ifndef PARALLELREADTIFF_H +#define PARALLELREADTIFF_H + +#include +#include +#include +#include +#include +#include +#include + +#include "tiffio.h" +#include "omp.h" +// #include "commons/generic/pdc_generic.h" + +typedef struct { + uint64_t *range, size_t length +} parallel_tiff_range_t; + +void parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tiff_range_t *strip_range); + +#endif // PARALLELREADTIFF_H \ No newline at end of file diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 34912b3f0..f5099af5e 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -1,6 +1,7 @@ #include #include #include +#include #define ENABLE_MPI 1 @@ -12,3 +13,49 @@ #include "pdc_client_server_common.h" #include "pdc_client_connect.h" +#include "llsm/parallelReadTiff.h" + + +void +parse_console_args(int argc, char *argv[], char **file_name) +{ + int c; + + while ((c = getopt(argc, argv, "f:")) != -1) { + switch (c) { + case 'f': + *filename = optarg; + break; + case '?': + if (optopt == 'f') { + fprintf(stderr, "Option -%c requires an argument.\n", optopt); + } + else { + fprintf(stderr, "Unknown option: -%c\n", optopt); + } + return 1; + default: + abort(); + } + } +} + + + +int +main(int argc, char *argv[]) +{ + + char * file_name = NULL; + void *tiff = NULL; + + parse_console_args(argc, argv, &file_name); + + printf("Filename: %s\n", file_name ? file_name : "(none)"); + + parallel_TIFF_load(file_name, &tiff, 1, NULL); + + + + return 0; +} \ No newline at end of file From c5314848cc7896c2d3e343d6d54acc1a5483c52f Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 23:41:49 -0500 Subject: [PATCH 083/216] llsm tiff import test --- tools/llsm_importer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index f5099af5e..81bd85b80 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -2,6 +2,7 @@ #include #include #include +#include #define ENABLE_MPI 1 @@ -55,7 +56,6 @@ main(int argc, char *argv[]) parallel_TIFF_load(file_name, &tiff, 1, NULL); - return 0; } \ No newline at end of file From b19f722e48d5d1b9a6c93554ea2d8180bd91d53c Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 23:43:03 -0500 Subject: [PATCH 084/216] llsm tiff import test --- tools/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 9e4062959..163df9dde 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -82,5 +82,6 @@ foreach(program ${PROGRAMS}) add_executable(${program} ${program}.c) target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) target_link_libraries(${program} cjson) + target_link_libraries(${program} llsm_tiff) endforeach(program) From 0cd72e8678f3d58706426a57c2a308d12556b7d1 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 23:45:07 -0500 Subject: [PATCH 085/216] llsm tiff import test --- tools/llsm/parallelReadTiff.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h index 5745216e0..02e0379a6 100644 --- a/tools/llsm/parallelReadTiff.h +++ b/tools/llsm/parallelReadTiff.h @@ -2,6 +2,7 @@ #define PARALLELREADTIFF_H #include +#include #include #include #include @@ -14,7 +15,8 @@ // #include "commons/generic/pdc_generic.h" typedef struct { - uint64_t *range, size_t length + uint64_t *range; + size_t length; } parallel_tiff_range_t; void parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tiff_range_t *strip_range); From 9670bc579b366ad3c988ee24b2177bc4efd0800e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 23:49:12 -0500 Subject: [PATCH 086/216] update code --- tools/llsm/parallelReadTiff.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index ef417ee60..43ad2f61f 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -668,7 +668,7 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u t = m; } } - z = s + 1; + *z = s + 1; } else { if (strip_range->length != 2) { @@ -676,7 +676,7 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u } else { *startSlice = (uint64_t) * (strip_range->range) - 1; - *z = (uint64_t) * (strip_range->range + 1)) - startSlice; + *z = (uint64_t) * (strip_range->range + 1) - startSlice; if (!TIFFSetDirectory(tif, startSlice[0] + z[0] - 1) || !TIFFSetDirectory(tif, startSlice[0])) { printf("tiff:rangeOutOfBound", "Range is out of bounds"); } @@ -687,7 +687,7 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u *imageJ_Z = imageJImGetZ(tif); if (*is_imageJ) { *is_imageJ = 1; - *imageJ_Z = imageJImGetZ(fileName); + *imageJ_Z = imageJImGetZ(tif); if (*imageJ_Z) *z = *imageJ_Z; } @@ -753,8 +753,8 @@ createDoubleArray(int ndim, size_t *dim) return array; } -void -_get_tiff_array(int bits, int ndim, size_t *dim) +void * +_get_tiff_array(int bits, int ndim, size_t *dims) { void *tiff = NULL; if (bits == 8) { @@ -774,7 +774,7 @@ _get_tiff_array(int bits, int ndim, size_t *dim) void _TIFF_load(char *fileName, uint64_t x, uint64_t y, uint64_t z, uint64_t bits, uint64_t startSlice, - uint64_t stripSize, uint8_t flipXY, int ndim, sizt_t *dims, void **tiff_ptr) + uint64_t stripSize, uint8_t flipXY, int ndim, size_t *dims, void **tiff_ptr) { if (tiff == NULL) { printf("tiff:dataTypeError, Data type not suppported\n"); @@ -799,7 +799,7 @@ parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tif { uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 0, is_imageJ = 0, imageJ_Z = 0; - get_tif_info(file_name, strip_range, &x, &y, &z, &bits, &startSlice, &stripeSize, &is_imageJ, &imageJ_Z); + get_tiff_info(file_name, strip_range, &x, &y, &z, &bits, &startSlice, &stripeSize, &is_imageJ, &imageJ_Z); int ndim = 3; uint64_t dims[ndim]; @@ -807,7 +807,7 @@ parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tif dims[1] = flipXY ? x : y; dims[2] = z; - _TIFF_load(fileName, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, tiff_ptr); + _TIFF_load(file_name, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, tiff_ptr); } // void From b2afd956c1035a20de4452db96cb5851fefeab55 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 23:52:19 -0500 Subject: [PATCH 087/216] update code --- tools/llsm/parallelReadTiff.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 43ad2f61f..2ab7f7333 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -773,15 +773,15 @@ _get_tiff_array(int bits, int ndim, size_t *dims) } void -_TIFF_load(char *fileName, uint64_t x, uint64_t y, uint64_t z, uint64_t bits, uint64_t startSlice, - uint64_t stripSize, uint8_t flipXY, int ndim, size_t *dims, void **tiff_ptr) +_TIFF_load(char *fileName, uint8_t isImageJIm, uint64_t x, uint64_t y, uint64_t z, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY, int ndim, size_t *dims, void **tiff_ptr) { - if (tiff == NULL) { + if (tiff_ptr == NULL) { printf("tiff:dataTypeError, Data type not suppported\n"); } *tiff_ptr = _get_tiff_array(bits, ndim, dims); // Case for ImageJ - if (imageJIm) { + if (isImageJIm) { readTiffParallelImageJ(x, y, z, fileName, *tiff_ptr, bits, startSlice, stripSize, flipXY); } // Case for 2D @@ -799,7 +799,7 @@ parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tif { uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 0, is_imageJ = 0, imageJ_Z = 0; - get_tiff_info(file_name, strip_range, &x, &y, &z, &bits, &startSlice, &stripeSize, &is_imageJ, &imageJ_Z); + get_tiff_info(fileName, strip_range, &x, &y, &z, &bits, &startSlice, &stripeSize, &is_imageJ, &imageJ_Z); int ndim = 3; uint64_t dims[ndim]; @@ -807,7 +807,7 @@ parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tif dims[1] = flipXY ? x : y; dims[2] = z; - _TIFF_load(file_name, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, tiff_ptr); + _TIFF_load(fileName, is_imageJ, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, tiff_ptr); } // void From 448383b2efdd513d2c53a0594d530efce9586c29 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 23:54:09 -0500 Subject: [PATCH 088/216] update code --- tools/llsm/parallelReadTiff.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 2ab7f7333..9cc97b17c 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -676,7 +676,7 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u } else { *startSlice = (uint64_t) * (strip_range->range) - 1; - *z = (uint64_t) * (strip_range->range + 1) - startSlice; + *z = (uint64_t) * (strip_range->range[1]) - startSlice; if (!TIFFSetDirectory(tif, startSlice[0] + z[0] - 1) || !TIFFSetDirectory(tif, startSlice[0])) { printf("tiff:rangeOutOfBound", "Range is out of bounds"); } @@ -882,7 +882,7 @@ parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tif // } // else { // startSlice = (uint64_t) * (mxGetPr(prhs[1])) - 1; -// z = (uint64_t) * ((mxGetPr(prhs[1]) + 1)) - startSlice; + // z = (uint64_t) * ((mxGetPr(prhs[1]) + 1)) - startSlice; // if (!TIFFSetDirectory(tif, startSlice + z - 1) || !TIFFSetDirectory(tif, startSlice)) { // printf("tiff:rangeOutOfBound", "Range is out of bounds"); // } From b79f950e71eb12d825fafb5240d4cc9a6ef437ac Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 23:55:25 -0500 Subject: [PATCH 089/216] update code --- tools/llsm/parallelReadTiff.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 9cc97b17c..35e53f331 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -675,8 +675,8 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u printf("tiff:inputError", "Input range is not 2"); } else { - *startSlice = (uint64_t) * (strip_range->range) - 1; - *z = (uint64_t) * (strip_range->range[1]) - startSlice; + *startSlice = (uint64_t) (*(strip_range->range)) - 1; + *z = (uint64_t) (*(strip_range->range + 1)) - startSlice; if (!TIFFSetDirectory(tif, startSlice[0] + z[0] - 1) || !TIFFSetDirectory(tif, startSlice[0])) { printf("tiff:rangeOutOfBound", "Range is out of bounds"); } From c2d5a3aea84b287104bc69400c2ae8450a05fe1d Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 23:57:10 -0500 Subject: [PATCH 090/216] update code --- tools/llsm/parallelReadTiff.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 35e53f331..1f0cedbbe 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -675,8 +675,8 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u printf("tiff:inputError", "Input range is not 2"); } else { - *startSlice = (uint64_t) (*(strip_range->range)) - 1; - *z = (uint64_t) (*(strip_range->range + 1)) - startSlice; + *startSlice = (uint64_t) ((strip_range->range[0])) - 1; + *z = (uint64_t) ((strip_range->range[1])) - startSlice; if (!TIFFSetDirectory(tif, startSlice[0] + z[0] - 1) || !TIFFSetDirectory(tif, startSlice[0])) { printf("tiff:rangeOutOfBound", "Range is out of bounds"); } From e18ebcf884cfb7bb43ec6d89f58f80724a16e5bc Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 13 Apr 2023 23:59:03 -0500 Subject: [PATCH 091/216] update code --- tools/llsm/parallelReadTiff.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 1f0cedbbe..a242eb3bc 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -675,8 +675,8 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u printf("tiff:inputError", "Input range is not 2"); } else { - *startSlice = (uint64_t) ((strip_range->range[0])) - 1; - *z = (uint64_t) ((strip_range->range[1])) - startSlice; + *startSlice = (uint64_t)(*(strip_range->range)) - 1; + *z = (uint64_t)(*(strip_range->range + 1)) - startSlice[0]; if (!TIFFSetDirectory(tif, startSlice[0] + z[0] - 1) || !TIFFSetDirectory(tif, startSlice[0])) { printf("tiff:rangeOutOfBound", "Range is out of bounds"); } @@ -882,7 +882,7 @@ parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tif // } // else { // startSlice = (uint64_t) * (mxGetPr(prhs[1])) - 1; - // z = (uint64_t) * ((mxGetPr(prhs[1]) + 1)) - startSlice; +// z = (uint64_t) * ((mxGetPr(prhs[1]) + 1)) - startSlice; // if (!TIFFSetDirectory(tif, startSlice + z - 1) || !TIFFSetDirectory(tif, startSlice)) { // printf("tiff:rangeOutOfBound", "Range is out of bounds"); // } From cfb4fde69a97186fc31db98a0b0d4ba07921a24f Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 14 Apr 2023 00:15:54 -0500 Subject: [PATCH 092/216] update code --- tools/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 163df9dde..84391ad4e 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -83,5 +83,6 @@ foreach(program ${PROGRAMS}) target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) target_link_libraries(${program} cjson) target_link_libraries(${program} llsm_tiff) + target_include_directories(${program} PUBLIC ${PDC_EXT_INCLUDE_DEPENDENCIES}) endforeach(program) From 1afb06ae6121818223984085b0f1d5ee6e592e60 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 14 Apr 2023 09:21:46 -0500 Subject: [PATCH 093/216] update code --- tools/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 84391ad4e..bce78df19 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -83,6 +83,6 @@ foreach(program ${PROGRAMS}) target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) target_link_libraries(${program} cjson) target_link_libraries(${program} llsm_tiff) - target_include_directories(${program} PUBLIC ${PDC_EXT_INCLUDE_DEPENDENCIES}) + target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) endforeach(program) From 912b14c085b43a2f02fcadccee116ca631d9031c Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 14 Apr 2023 09:30:44 -0500 Subject: [PATCH 094/216] update code --- tools/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index bce78df19..fd4736045 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -81,6 +81,7 @@ set(PROGRAMS foreach(program ${PROGRAMS}) add_executable(${program} ${program}.c) target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) + target_link_libraries(${program} pdc) target_link_libraries(${program} cjson) target_link_libraries(${program} llsm_tiff) target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) From 3ae5c1412e70e86095e058b767cf862efabcc0bb Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 14 Apr 2023 09:31:50 -0500 Subject: [PATCH 095/216] update code --- tools/llsm_importer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 81bd85b80..71f61e4d9 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -11,8 +11,8 @@ #endif #include "pdc.h" -#include "pdc_client_server_common.h" -#include "pdc_client_connect.h" +// #include "pdc_client_server_common.h" +// #include "pdc_client_connect.h" #include "llsm/parallelReadTiff.h" From 16c5843278b132d596ffda47f9b78373db32d52e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 14 Apr 2023 09:35:08 -0500 Subject: [PATCH 096/216] update code --- tools/llsm_importer.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 71f61e4d9..e6c156b6b 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -16,8 +16,7 @@ #include "llsm/parallelReadTiff.h" - -void +int parse_console_args(int argc, char *argv[], char **file_name) { int c; @@ -25,7 +24,7 @@ parse_console_args(int argc, char *argv[], char **file_name) while ((c = getopt(argc, argv, "f:")) != -1) { switch (c) { case 'f': - *filename = optarg; + *file_name = optarg; break; case '?': if (optopt == 'f') { @@ -41,21 +40,22 @@ parse_console_args(int argc, char *argv[], char **file_name) } } - - int main(int argc, char *argv[]) { - char * file_name = NULL; - void *tiff = NULL; + char *file_name = NULL; + void *tiff = NULL; - parse_console_args(argc, argv, &file_name); + int parse_code = parse_console_args(argc, argv, &file_name); + + if (parse_code) { + return parse_code; + } printf("Filename: %s\n", file_name ? file_name : "(none)"); parallel_TIFF_load(file_name, &tiff, 1, NULL); - return 0; } \ No newline at end of file From b4ff1c63f81b06346ce1d29148ef376fef7f21fc Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 09:28:17 -0500 Subject: [PATCH 097/216] update cmake and llsm_importer --- tools/CMakeLists.txt | 29 +++++++++++++++++++++++------ tools/llsm/CMakeLists.txt | 0 tools/llsm_importer.c | 12 +++++++++--- tools/pdc_export.c | 2 +- tools/pdc_import.c | 2 +- 5 files changed, 34 insertions(+), 11 deletions(-) delete mode 100644 tools/llsm/CMakeLists.txt diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index fd4736045..77ad41813 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -64,18 +64,12 @@ endif() add_definitions(-DENABLE_MPI=1) add_library(cjson cjson/cJSON.c) -add_library(llsm_tiff llsm/parallelReadTiff.c) -target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS}) -target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/llsm) -target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES}) -# add_subdirectory(llsm) set(PROGRAMS pdc_ls pdc_import pdc_export - llsm_importer ) foreach(program ${PROGRAMS}) @@ -87,3 +81,26 @@ foreach(program ${PROGRAMS}) target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) endforeach(program) + +# Find LibTIFF +option(USE_LIB_TIFF "Enable LibTiff." ON) +if(USE_LIB_TIFF) + find_package(TIFF REQUIRED) + if(TIFF_FOUND) + # Add the LibTIFF include directory to the include path + include_directories(${TIFF_INCLUDE_DIRS}) + add_library(llsm_tiff llsm/parallelReadTiff.c) + target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS}) + target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/llsm) + target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES}) + target_link_libraries(llsm_tiff ${TIFF_LIBRARIES}) + + add_executable(llsm_importer llsm_importer.c) + target_link_libraries(llsm_importer ${PDC_EXT_LIB_DEPENDENCIES}) + target_link_libraries(llsm_importer pdc) + target_link_libraries(llsm_importer cjson) + target_link_libraries(llsm_importer ${TIFF_LIBRARIES}) + target_link_libraries(llsm_importer llsm_tiff) + else() + message(WARNING "LibTiff not found, ignore building the executables which requires LibTiff support.") + endif() \ No newline at end of file diff --git a/tools/llsm/CMakeLists.txt b/tools/llsm/CMakeLists.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index e6c156b6b..096cc38ff 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -4,10 +4,10 @@ #include #include -#define ENABLE_MPI 1 +// #define ENABLE_MPI 1 #ifdef ENABLE_MPI -#include "mpi.h" +// #include "mpi.h" #endif #include "pdc.h" @@ -46,7 +46,8 @@ main(int argc, char *argv[]) char *file_name = NULL; void *tiff = NULL; - + int i = 0; + char bytes[10]; int parse_code = parse_console_args(argc, argv, &file_name); if (parse_code) { @@ -57,5 +58,10 @@ main(int argc, char *argv[]) parallel_TIFF_load(file_name, &tiff, 1, NULL); + for (i = 0; i < 10; i++) { + bytes[i] = (char)tiff[i]; + } + printf("first few bytes : %s\n", bytes); + return 0; } \ No newline at end of file diff --git a/tools/pdc_export.c b/tools/pdc_export.c index e46c17f2a..af42e116a 100644 --- a/tools/pdc_export.c +++ b/tools/pdc_export.c @@ -8,7 +8,7 @@ #include #include "hdf5.h" -#define ENABLE_MPI 1 +// #define ENABLE_MPI 1 #ifdef ENABLE_MPI #include "mpi.h" diff --git a/tools/pdc_import.c b/tools/pdc_import.c index 9e12d3a84..9388ac495 100644 --- a/tools/pdc_import.c +++ b/tools/pdc_import.c @@ -2,7 +2,7 @@ #include #include -#define ENABLE_MPI 1 +// #define ENABLE_MPI 1 #ifdef ENABLE_MPI #include "mpi.h" From 84804c8ce98d37b2e6b880dd446a2b157f5c2dad Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 09:28:53 -0500 Subject: [PATCH 098/216] update cmake and llsm_importer --- tools/CMakeLists.txt | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 77ad41813..38858e3fa 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -66,20 +66,20 @@ add_definitions(-DENABLE_MPI=1) add_library(cjson cjson/cJSON.c) -set(PROGRAMS - pdc_ls - pdc_import - pdc_export - ) +# set(PROGRAMS +# pdc_ls +# pdc_import +# pdc_export +# ) -foreach(program ${PROGRAMS}) - add_executable(${program} ${program}.c) - target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) - target_link_libraries(${program} pdc) - target_link_libraries(${program} cjson) - target_link_libraries(${program} llsm_tiff) - target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) -endforeach(program) +# foreach(program ${PROGRAMS}) +# add_executable(${program} ${program}.c) +# target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) +# target_link_libraries(${program} pdc) +# target_link_libraries(${program} cjson) +# target_link_libraries(${program} llsm_tiff) +# target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) +# endforeach(program) # Find LibTIFF From 95194c97f94ae0ac11dbdcdc1604758c834d7e7b Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 09:32:03 -0500 Subject: [PATCH 099/216] close if in cmake --- tools/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 38858e3fa..ea6a8fdf3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -103,4 +103,5 @@ if(USE_LIB_TIFF) target_link_libraries(llsm_importer llsm_tiff) else() message(WARNING "LibTiff not found, ignore building the executables which requires LibTiff support.") - endif() \ No newline at end of file + endif() +endif() \ No newline at end of file From 12ac52b6103c3a64a2422d14fa16a09a80ecf4e5 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 09:41:56 -0500 Subject: [PATCH 100/216] cmake fix tiff --- tools/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index ea6a8fdf3..f4513a6a6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -89,11 +89,12 @@ if(USE_LIB_TIFF) if(TIFF_FOUND) # Add the LibTIFF include directory to the include path include_directories(${TIFF_INCLUDE_DIRS}) - add_library(llsm_tiff llsm/parallelReadTiff.c) + add_library(llsm_tiff llsm/parallelReadTiff.c llsm/parallelReadTiff.h) target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS}) - target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/llsm) target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES}) - target_link_libraries(llsm_tiff ${TIFF_LIBRARIES}) + target_link_libraries(llsm_tiff PUBLIC ${TIFF_LIBRARIES}) + target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/llsm) + add_executable(llsm_importer llsm_importer.c) target_link_libraries(llsm_importer ${PDC_EXT_LIB_DEPENDENCIES}) From a5066b8d7cde6967e39f930ef12ab93e00950209 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 09:49:33 -0500 Subject: [PATCH 101/216] cmake policy to suppress warning --- tools/CMakeLists.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index f4513a6a6..79c327367 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,5 +1,24 @@ cmake_minimum_required (VERSION 2.8.12) +# Setup cmake policies. +foreach(p + CMP0012 + CMP0013 + CMP0014 + CMP0022 # CMake 2.8.12 + CMP0025 # CMake 3.0 + CMP0053 # CMake 3.1 + CMP0054 # CMake 3.1 + CMP0074 # CMake 3.12 + CMP0075 # CMake 3.12 + CMP0083 # CMake 3.14 + CMP0093 # CMake 3.15 + ) + if(POLICY ${p}) + cmake_policy(SET ${p} NEW) + endif() +endforeach() + project(PDC_VOL C) include_directories( From a66dd0d9f9a3c7e30296d5572926f94483e8a183 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 10:20:26 -0500 Subject: [PATCH 102/216] add pdc include dir --- tools/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 79c327367..6145514ef 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -121,6 +121,7 @@ if(USE_LIB_TIFF) target_link_libraries(llsm_importer cjson) target_link_libraries(llsm_importer ${TIFF_LIBRARIES}) target_link_libraries(llsm_importer llsm_tiff) + target_include_directories(llsm_tiff PUBLIC ${PDC_INCLUDE_DIR}) else() message(WARNING "LibTiff not found, ignore building the executables which requires LibTiff support.") endif() From 372cc4f404cc4b6e0096db5cf167c1f419748d85 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 10:23:58 -0500 Subject: [PATCH 103/216] update code --- tools/llsm_importer.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 096cc38ff..b9dcdfcbe 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -48,6 +48,7 @@ main(int argc, char *argv[]) void *tiff = NULL; int i = 0; char bytes[10]; + char *tiff_str_ptr; int parse_code = parse_console_args(argc, argv, &file_name); if (parse_code) { @@ -58,8 +59,9 @@ main(int argc, char *argv[]) parallel_TIFF_load(file_name, &tiff, 1, NULL); + tiff_str_ptr = (char *)tiff; for (i = 0; i < 10; i++) { - bytes[i] = (char)tiff[i]; + bytes[i] = tiff_str_ptr[i]; } printf("first few bytes : %s\n", bytes); From f36920d3037475fb0899fe96910392dbf9d7c79c Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 10:28:53 -0500 Subject: [PATCH 104/216] update code --- tools/llsm_importer.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index b9dcdfcbe..60e737d22 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -22,6 +22,7 @@ parse_console_args(int argc, char *argv[], char **file_name) int c; while ((c = getopt(argc, argv, "f:")) != -1) { + printf("c : %c \n", c); switch (c) { case 'f': *file_name = optarg; @@ -49,8 +50,9 @@ main(int argc, char *argv[]) int i = 0; char bytes[10]; char *tiff_str_ptr; + printf("Program started!\n"); int parse_code = parse_console_args(argc, argv, &file_name); - + printf("parse_code %d\n", parse_code); if (parse_code) { return parse_code; } From a80d9dfe132de012ff90dfa7f0d634600b16cd44 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 10:34:00 -0500 Subject: [PATCH 105/216] update code --- tools/llsm_importer.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 60e737d22..258a4af76 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -27,16 +27,9 @@ parse_console_args(int argc, char *argv[], char **file_name) case 'f': *file_name = optarg; break; - case '?': - if (optopt == 'f') { - fprintf(stderr, "Option -%c requires an argument.\n", optopt); - } - else { - fprintf(stderr, "Unknown option: -%c\n", optopt); - } - return 1; default: - abort(); + fprintf(stderr, "Usage: %s [-f filename]\n", argv[0]); + exit(EXIT_FAILURE); } } } From 0b188f72e6103e942513b7ae0b3f4f53dceffb01 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 10:35:47 -0500 Subject: [PATCH 106/216] update code --- tools/llsm_importer.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 258a4af76..0d79805b7 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -19,19 +19,22 @@ int parse_console_args(int argc, char *argv[], char **file_name) { - int c; - + int c, parse_code = -1; + while ((c = getopt(argc, argv, "f:")) != -1) { printf("c : %c \n", c); switch (c) { case 'f': *file_name = optarg; + parse_code = 1; break; default: fprintf(stderr, "Usage: %s [-f filename]\n", argv[0]); + parse_code = -1; exit(EXIT_FAILURE); } } + return parse_code; } int From b54fc25168b507728eef49fab61f88455dbd476e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 10:36:35 -0500 Subject: [PATCH 107/216] update code --- tools/llsm_importer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 0d79805b7..2a47c561a 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -26,7 +26,7 @@ parse_console_args(int argc, char *argv[], char **file_name) switch (c) { case 'f': *file_name = optarg; - parse_code = 1; + parse_code = 0; break; default: fprintf(stderr, "Usage: %s [-f filename]\n", argv[0]); From f665f7d98613988275badcbfd2201a36165142ad Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 10:56:11 -0500 Subject: [PATCH 108/216] update code --- tools/llsm/parallelReadTiff.c | 33 +++++++++++++++++++++++++++++++++ tools/llsm/parallelReadTiff.h | 3 --- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index a242eb3bc..ed1f7674f 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -1,4 +1,11 @@ #include "parallelReadTiff.h" +#include "tiffio.h" + +#define USE_OMP 0 + +#ifdef USE_OMP +#include "omp.h" +#endif void DummyHandler(const char *module, const char *fmt, va_list ap) @@ -16,7 +23,9 @@ readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, vo uint64_t bytes = bits / 8; int32_t w; +#ifdef USE_OMP #pragma omp parallel for +#endif for (w = 0; w < numWorkers; w++) { TIFF *tif = TIFFOpen(fileName, "r"); @@ -97,7 +106,9 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void char errString[10000]; if (compressed > 1 || z < 32768) { TIFFClose(tif); +#ifdef USE_OMP #pragma omp parallel for +#endif for (w = 0; w < numWorkers; w++) { uint8_t outCounter = 0; @@ -105,7 +116,9 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void while (!tif) { tif = TIFFOpen(fileName, "r"); if (outCounter == 3) { +#ifdef USE_OMP #pragma omp critical +#endif { err = 1; sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); @@ -125,7 +138,9 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void while (!TIFFSetDirectory(tif, (uint64_t)dir) && counter < 3) { counter++; if (counter == 3) { +#ifdef USE_OMP #pragma omp critical +#endif { err = 1; sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); @@ -139,7 +154,9 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void // loading the data into a buffer int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize * x * bytes); if (cBytes < 0) { +#ifdef USE_OMP #pragma omp critical +#endif { errBak = 1; err = 1; @@ -249,7 +266,9 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void uint64_t size = x * y * z * (bits / 8); void * tiffC = malloc(size); memcpy(tiffC, tiff, size); +#ifdef USE_OMP #pragma omp parallel for +#endif for (uint64_t k = 0; k < z; k++) { for (uint64_t j = 0; j < x; j++) { for (uint64_t i = 0; i < y; i++) { @@ -294,7 +313,9 @@ readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, uint64_t bytes = bits / 8; int32_t w; +#ifdef USE_OMP #pragma omp parallel for +#endif for (w = 0; w < numWorkers; w++) { TIFF *tif = TIFFOpen(fileName, "r"); @@ -373,7 +394,9 @@ readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, voi uint8_t errBak = 0; char errString[10000]; +#ifdef USE_OMP #pragma omp parallel for +#endif for (w = 0; w < numWorkers; w++) { uint8_t outCounter = 0; @@ -381,7 +404,9 @@ readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, voi while (!tif) { tif = TIFFOpen(fileName, "r"); if (outCounter == 3) { +#ifdef USE_OMP #pragma omp critical +#endif { err = 1; sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); @@ -399,7 +424,9 @@ readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, voi counter + 1); counter++; if (counter == 3) { +#ifdef USE_OMP #pragma omp critical +#endif { err = 1; sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); @@ -412,7 +439,9 @@ readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, voi // loading the data into a buffer int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize * x * bytes); if (cBytes < 0) { +#ifdef USE_OMP #pragma omp critical +#endif { errBak = 1; err = 1; @@ -532,7 +561,9 @@ readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char *fileName, // Swap endianess for types greater than 8 bits // TODO: May need to change later because we may not always need to swap if (bits > 8) { +#ifdef USE_OMP #pragma omp parallel for +#endif for (uint64_t i = 0; i < x * y * z; i++) { switch (bits) { case 16: @@ -570,7 +601,9 @@ readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char *fileName, uint64_t size = x * y * z * (bits / 8); void * tiffC = malloc(size); memcpy(tiffC, tiff, size); +#ifdef USE_OMP #pragma omp parallel for +#endif for (uint64_t k = 0; k < z; k++) { for (uint64_t j = 0; j < x; j++) { for (uint64_t i = 0; i < y; i++) { diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h index 02e0379a6..e8fe59fbc 100644 --- a/tools/llsm/parallelReadTiff.h +++ b/tools/llsm/parallelReadTiff.h @@ -10,9 +10,6 @@ #include #include -#include "tiffio.h" -#include "omp.h" -// #include "commons/generic/pdc_generic.h" typedef struct { uint64_t *range; From ba2f67a27c7fde27f927364808ca321bf0c68581 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 23:51:27 -0500 Subject: [PATCH 109/216] update array generating method --- tools/llsm/parallelReadTiff.c | 95 ++++++++++++++++++++++------------- tools/llsm/parallelReadTiff.h | 8 +++ 2 files changed, 67 insertions(+), 36 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index ed1f7674f..712c0d7e7 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -730,60 +730,83 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u TIFFClose(tif); } -uint8_t *** +uint8_t * createU8Array(int ndim, size_t *dim) { - size_t i, j; - uint8_t ***array = (uint8_t ***)malloc(dim[0] * sizeof(uint8_t **)); - for (i = 0; i < dim[0]; i++) { - array[i] = (uint8_t **)malloc(dim[1] * sizeof(uint8_t *)); - for (j = 0; j < dim[1]; j++) { - array[i][j] = (uint8_t *)calloc(dim[2], sizeof(uint8_t)); - } + size_t i = 0, dim_prod = 1; + for (i = 0; i < ndim; i++) { + dim_prod *= dim[i]; } - return array; + return (uint8_t*)malloc(dim_prod * sizeof(uint8_t)); + // size_t i, j; + // uint8_t ***array = (uint8_t ***)malloc(dim[0] * sizeof(uint8_t **)); + // for (i = 0; i < dim[0]; i++) { + // array[i] = (uint8_t **)malloc(dim[1] * sizeof(uint8_t *)); + // for (j = 0; j < dim[1]; j++) { + // array[i][j] = (uint8_t *)calloc(dim[2], sizeof(uint8_t)); + // } + // } + // return array; } -uint16_t *** +uint16_t * createU16Array(int ndim, size_t *dim) { - size_t i, j; - uint16_t ***array = (uint16_t ***)malloc(dim[0] * sizeof(uint16_t **)); - for (i = 0; i < dim[0]; i++) { - array[i] = (uint16_t **)malloc(dim[1] * sizeof(uint16_t *)); - for (j = 0; j < dim[1]; j++) { - array[i][j] = (uint16_t *)calloc(dim[2], sizeof(uint16_t)); - } + size_t i = 0, dim_prod = 1; + for (i = 0; i < ndim; i++) { + dim_prod *= dim[i]; } - return array; + return (uint16_t*)malloc(dim_prod * sizeof(uint16_t)); + + // size_t i, j; + // uint16_t ***array = (uint16_t ***)malloc(dim[0] * sizeof(uint16_t **)); + // for (i = 0; i < dim[0]; i++) { + // array[i] = (uint16_t **)malloc(dim[1] * sizeof(uint16_t *)); + // for (j = 0; j < dim[1]; j++) { + // array[i][j] = (uint16_t *)calloc(dim[2], sizeof(uint16_t)); + // } + // } + // return array; } -float *** +float * createFloatArray(int ndim, size_t *dim) { - size_t i, j; - float ***array = (float ***)malloc(dim[0] * sizeof(float **)); - for (i = 0; i < dim[0]; i++) { - array[i] = (float **)malloc(dim[1] * sizeof(float *)); - for (j = 0; j < dim[1]; j++) { - array[i][j] = (float *)calloc(dim[2], sizeof(float)); - } + size_t i = 0, dim_prod = 1; + for (i = 0; i < ndim; i++) { + dim_prod *= dim[i]; } - return array; + return (float *)malloc(dim_prod * sizeof(float)); + + // size_t i, j; + // float ***array = (float ***)malloc(dim[0] * sizeof(float **)); + // for (i = 0; i < dim[0]; i++) { + // array[i] = (float **)malloc(dim[1] * sizeof(float *)); + // for (j = 0; j < dim[1]; j++) { + // array[i][j] = (float *)calloc(dim[2], sizeof(float)); + // } + // } + // return array; } -double *** +double * createDoubleArray(int ndim, size_t *dim) { - size_t i, j; - double ***array = (double ***)malloc(dim[0] * sizeof(double **)); - for (i = 0; i < dim[0]; i++) { - array[i] = (double **)malloc(dim[1] * sizeof(double *)); - for (j = 0; j < dim[1]; j++) { - array[i][j] = (double *)calloc(dim[2], sizeof(double)); - } + size_t i = 0, dim_prod = 1; + for (i = 0; i < ndim; i++) { + dim_prod *= dim[i]; } - return array; + return (double *)malloc(dim_prod * sizeof(double)); + + // size_t i, j; + // double ***array = (double ***)malloc(dim[0] * sizeof(double **)); + // for (i = 0; i < dim[0]; i++) { + // array[i] = (double **)malloc(dim[1] * sizeof(double *)); + // for (j = 0; j < dim[1]; j++) { + // array[i][j] = (double *)calloc(dim[2], sizeof(double)); + // } + // } + // return array; } void * diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h index e8fe59fbc..f7b4ba2c5 100644 --- a/tools/llsm/parallelReadTiff.h +++ b/tools/llsm/parallelReadTiff.h @@ -11,6 +11,14 @@ #include +#define CREATE_ARRAY(result_var, type, ndim, dim) do { \ + size_t i = 0, dim_prod = 1; \ + for (i = 0; i < (ndim); i++) { \ + dim_prod *= (dim)[i]; \ + } \ + result_var = (void *)malloc(dim_prod * sizeof(type)); \ +} while (0) + typedef struct { uint64_t *range; size_t length; From 08ab57e98a297cd805951b965f59d0f82f10d35e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 23:55:57 -0500 Subject: [PATCH 110/216] update array generating method --- tools/llsm/parallelReadTiff.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 712c0d7e7..ac0040da8 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -814,16 +814,20 @@ _get_tiff_array(int bits, int ndim, size_t *dims) { void *tiff = NULL; if (bits == 8) { - tiff = (void *)createU8Array(ndim, dims); + CREATE_ARRAY(tiff, uint8_t, ndim, dims); + // tiff = (void *)createU8Array(ndim, dims); } else if (bits == 16) { - tiff = (void *)createU16Array(ndim, dims); + CREATE_ARRAY(tiff, uint16_t, ndim, dims); + // tiff = (void *)createU16Array(ndim, dims); } else if (bits == 32) { - tiff = (void *)createFloatArray(ndim, dims); + CREATE_ARRAY(tiff, float, ndim, dims); + // tiff = (void *)createFloatArray(ndim, dims); } else if (bits == 64) { - tiff = (void *)createDoubleArray(ndim, dims); + CREATE_ARRAY(tiff, double, ndim, dims); + // tiff = (void *)createDoubleArray(ndim, dims); } return tiff; } From 2377bb3e7674974add11e6450835e1eca28a02d9 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 19 Apr 2023 23:57:52 -0500 Subject: [PATCH 111/216] update array generating method --- tools/llsm/parallelReadTiff.c | 270 ++-------------------------------- tools/llsm/parallelReadTiff.h | 9 -- 2 files changed, 9 insertions(+), 270 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index ac0040da8..69178bdbd 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -7,6 +7,14 @@ #include "omp.h" #endif +#define CREATE_ARRAY(result_var, type, ndim, dim) do { \ + size_t i = 0, dim_prod = 1; \ + for (i = 0; i < (ndim); i++) { \ + dim_prod *= (dim)[i]; \ + } \ + result_var = (void *)malloc(dim_prod * sizeof(type)); \ +} while (0) + void DummyHandler(const char *module, const char *fmt, va_list ap) { @@ -730,104 +738,21 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u TIFFClose(tif); } -uint8_t * -createU8Array(int ndim, size_t *dim) -{ - size_t i = 0, dim_prod = 1; - for (i = 0; i < ndim; i++) { - dim_prod *= dim[i]; - } - return (uint8_t*)malloc(dim_prod * sizeof(uint8_t)); - // size_t i, j; - // uint8_t ***array = (uint8_t ***)malloc(dim[0] * sizeof(uint8_t **)); - // for (i = 0; i < dim[0]; i++) { - // array[i] = (uint8_t **)malloc(dim[1] * sizeof(uint8_t *)); - // for (j = 0; j < dim[1]; j++) { - // array[i][j] = (uint8_t *)calloc(dim[2], sizeof(uint8_t)); - // } - // } - // return array; -} - -uint16_t * -createU16Array(int ndim, size_t *dim) -{ - size_t i = 0, dim_prod = 1; - for (i = 0; i < ndim; i++) { - dim_prod *= dim[i]; - } - return (uint16_t*)malloc(dim_prod * sizeof(uint16_t)); - - // size_t i, j; - // uint16_t ***array = (uint16_t ***)malloc(dim[0] * sizeof(uint16_t **)); - // for (i = 0; i < dim[0]; i++) { - // array[i] = (uint16_t **)malloc(dim[1] * sizeof(uint16_t *)); - // for (j = 0; j < dim[1]; j++) { - // array[i][j] = (uint16_t *)calloc(dim[2], sizeof(uint16_t)); - // } - // } - // return array; -} - -float * -createFloatArray(int ndim, size_t *dim) -{ - size_t i = 0, dim_prod = 1; - for (i = 0; i < ndim; i++) { - dim_prod *= dim[i]; - } - return (float *)malloc(dim_prod * sizeof(float)); - - // size_t i, j; - // float ***array = (float ***)malloc(dim[0] * sizeof(float **)); - // for (i = 0; i < dim[0]; i++) { - // array[i] = (float **)malloc(dim[1] * sizeof(float *)); - // for (j = 0; j < dim[1]; j++) { - // array[i][j] = (float *)calloc(dim[2], sizeof(float)); - // } - // } - // return array; -} - -double * -createDoubleArray(int ndim, size_t *dim) -{ - size_t i = 0, dim_prod = 1; - for (i = 0; i < ndim; i++) { - dim_prod *= dim[i]; - } - return (double *)malloc(dim_prod * sizeof(double)); - - // size_t i, j; - // double ***array = (double ***)malloc(dim[0] * sizeof(double **)); - // for (i = 0; i < dim[0]; i++) { - // array[i] = (double **)malloc(dim[1] * sizeof(double *)); - // for (j = 0; j < dim[1]; j++) { - // array[i][j] = (double *)calloc(dim[2], sizeof(double)); - // } - // } - // return array; -} - void * _get_tiff_array(int bits, int ndim, size_t *dims) { void *tiff = NULL; if (bits == 8) { CREATE_ARRAY(tiff, uint8_t, ndim, dims); - // tiff = (void *)createU8Array(ndim, dims); } else if (bits == 16) { CREATE_ARRAY(tiff, uint16_t, ndim, dims); - // tiff = (void *)createU16Array(ndim, dims); } else if (bits == 32) { CREATE_ARRAY(tiff, float, ndim, dims); - // tiff = (void *)createFloatArray(ndim, dims); } else if (bits == 64) { CREATE_ARRAY(tiff, double, ndim, dims); - // tiff = (void *)createDoubleArray(ndim, dims); } return tiff; } @@ -868,181 +793,4 @@ parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tif dims[2] = z; _TIFF_load(fileName, is_imageJ, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, tiff_ptr); -} - -// void -// mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) -// { -// // Check if the fileName is a char array or matlab style -// char *fileName = NULL; -// if (!mxIsClass(prhs[0], "string")) { -// if (!mxIsChar(prhs[0])) -// printf("tiff:inputError", "The first argument must be a string"); -// fileName = mxArrayToString(prhs[0]); -// } -// else { -// mxArray *mString[1]; -// mxArray *mCharA[1]; - -// // Convert string to char array -// mString[0] = mxDuplicateArray(prhs[0]); -// mexCallMATLAB(1, mCharA, 1, mString, "char"); -// fileName = mxArrayToString(mCharA[0]); -// } - -// // Handle the tilde character in filenames on Linux/Mac -// // #ifndef _WIN32 -// // if(strchr(fileName,'~')) fileName = expandTilde(fileName); -// // #endif - -// uint8_t flipXY = 1; -// // uint8_t flipXY = 0; - -// // if(nrhs > 2){ -// // flipXY = (uint8_t)*(mxGetPr(prhs[2])); -// //} - -// TIFFSetWarningHandler(DummyHandler); -// TIFF *tif = TIFFOpen(fileName, "r"); -// if (!tif) -// printf("tiff:inputError", "File \"%s\" cannot be opened", fileName); - -// uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0; -// TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, &x); -// TIFFGetField(tif, TIFFTAG_IMAGELENGTH, &y); - -// if (nrhs == 1) { -// uint16_t s = 0, m = 0, t = 1; -// while (TIFFSetDirectory(tif, t)) { -// s = t; -// t *= 8; -// if (s > t) { -// t = 65535; -// printf("Number of slices > 32768\n"); -// break; -// } -// } -// while (s != t) { -// m = (s + t + 1) / 2; -// if (TIFFSetDirectory(tif, m)) { -// s = m; -// } -// else { -// if (m > 0) -// t = m - 1; -// else -// t = m; -// } -// } -// z = s + 1; -// } -// else { -// if (mxGetN(prhs[1]) != 2) { -// printf("tiff:inputError", "Input range is not 2"); -// } -// else { -// startSlice = (uint64_t) * (mxGetPr(prhs[1])) - 1; -// z = (uint64_t) * ((mxGetPr(prhs[1]) + 1)) - startSlice; -// if (!TIFFSetDirectory(tif, startSlice + z - 1) || !TIFFSetDirectory(tif, startSlice)) { -// printf("tiff:rangeOutOfBound", "Range is out of bounds"); -// } -// } -// } - -// TIFFGetField(tif, TIFFTAG_BITSPERSAMPLE, &bits); -// uint64_t stripSize = 1; -// TIFFGetField(tif, TIFFTAG_ROWSPERSTRIP, &stripSize); -// TIFFClose(tif); - -// uint8_t imageJIm = 0; -// if (isImageJIm(fileName)) { -// imageJIm = 1; -// uint64_t tempZ = imageJImGetZ(fileName); -// if (tempZ) -// z = tempZ; -// } - -// uint64_t dim[3]; -// dim[0] = y; -// dim[1] = x; -// dim[2] = z; - -// // Case for ImageJ -// if (imageJIm) { -// if (bits == 8) { -// plhs[0] = mxCreateNumericArray(3, dim, mxUINT8_CLASS, mxREAL); -// uint8_t *tiff = (uint8_t *)mxGetPr(plhs[0]); -// readTiffParallelImageJ(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else if (bits == 16) { -// plhs[0] = mxCreateNumericArray(3, dim, mxUINT16_CLASS, mxREAL); -// uint16_t *tiff = (uint16_t *)mxGetPr(plhs[0]); -// readTiffParallelImageJ(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else if (bits == 32) { -// plhs[0] = mxCreateNumericArray(3, dim, mxSINGLE_CLASS, mxREAL); -// float *tiff = (float *)mxGetPr(plhs[0]); -// readTiffParallelImageJ(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else if (bits == 64) { -// plhs[0] = mxCreateNumericArray(3, dim, mxDOUBLE_CLASS, mxREAL); -// double *tiff = (double *)mxGetPr(plhs[0]); -// readTiffParallelImageJ(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else { -// printf("tiff:dataTypeError", "Data type not suppported"); -// } -// } -// // Case for 2D -// else if (z <= 1) { -// if (bits == 8) { -// plhs[0] = mxCreateNumericArray(3, dim, mxUINT8_CLASS, mxREAL); -// uint8_t *tiff = (uint8_t *)mxGetPr(plhs[0]); -// readTiffParallel2D(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else if (bits == 16) { -// plhs[0] = mxCreateNumericArray(3, dim, mxUINT16_CLASS, mxREAL); -// uint16_t *tiff = (uint16_t *)mxGetPr(plhs[0]); -// readTiffParallel2D(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else if (bits == 32) { -// plhs[0] = mxCreateNumericArray(3, dim, mxSINGLE_CLASS, mxREAL); -// float *tiff = (float *)mxGetPr(plhs[0]); -// readTiffParallel2D(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else if (bits == 64) { -// plhs[0] = mxCreateNumericArray(3, dim, mxDOUBLE_CLASS, mxREAL); -// double *tiff = (double *)mxGetPr(plhs[0]); -// readTiffParallel2D(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else { -// printf("tiff:dataTypeError", "Data type not suppported"); -// } -// } -// // Case for 3D -// else { -// if (bits == 8) { -// plhs[0] = mxCreateNumericArray(3, dim, mxUINT8_CLASS, mxREAL); -// uint8_t *tiff = (uint8_t *)mxGetPr(plhs[0]); -// readTiffParallel(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else if (bits == 16) { -// plhs[0] = mxCreateNumericArray(3, dim, mxUINT16_CLASS, mxREAL); -// uint16_t *tiff = (uint16_t *)mxGetPr(plhs[0]); -// readTiffParallel(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else if (bits == 32) { -// plhs[0] = mxCreateNumericArray(3, dim, mxSINGLE_CLASS, mxREAL); -// float *tiff = (float *)mxGetPr(plhs[0]); -// readTiffParallel(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else if (bits == 64) { -// plhs[0] = mxCreateNumericArray(3, dim, mxDOUBLE_CLASS, mxREAL); -// double *tiff = (double *)mxGetPr(plhs[0]); -// readTiffParallel(x, y, z, fileName, (void *)tiff, bits, startSlice, stripSize, flipXY); -// } -// else { -// printf("tiff:dataTypeError", "Data type not suppported"); -// } -// } -// } \ No newline at end of file +} \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h index f7b4ba2c5..4f1d7ad5e 100644 --- a/tools/llsm/parallelReadTiff.h +++ b/tools/llsm/parallelReadTiff.h @@ -10,15 +10,6 @@ #include #include - -#define CREATE_ARRAY(result_var, type, ndim, dim) do { \ - size_t i = 0, dim_prod = 1; \ - for (i = 0; i < (ndim); i++) { \ - dim_prod *= (dim)[i]; \ - } \ - result_var = (void *)malloc(dim_prod * sizeof(type)); \ -} while (0) - typedef struct { uint64_t *range; size_t length; From da917802c2dc8e2da195cc31d224142d9a0203f2 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 20 Apr 2023 00:13:06 -0500 Subject: [PATCH 112/216] update array generating method --- tools/CMakeLists.txt | 2 ++ tools/llsm/parallelReadTiff.c | 30 +++++++++++++++--------------- tools/llsm_importer.c | 6 +++--- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 6145514ef..b6123afc3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -74,6 +74,8 @@ option(USE_SYSTEM_OPENMP "Use system-installed OpenMP." ON) if(USE_SYSTEM_OPENMP) find_package(OpenMP REQUIRED) if(OPENMP_FOUND) + add_definitions(-DENABLE_OPENMP=1) + set(ENABLE_OPENMP 1) set(OPENMP_LIBRARIES "${OpenMP_C_LIBRARIES}") else() message(FATAL_ERROR "OpenMP not found") diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 69178bdbd..0a8f609fd 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -1,9 +1,9 @@ #include "parallelReadTiff.h" #include "tiffio.h" -#define USE_OMP 0 +#define ENABLE_OPENMP -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #include "omp.h" #endif @@ -31,7 +31,7 @@ readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, vo uint64_t bytes = bits / 8; int32_t w; -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp parallel for #endif for (w = 0; w < numWorkers; w++) { @@ -114,7 +114,7 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void char errString[10000]; if (compressed > 1 || z < 32768) { TIFFClose(tif); -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp parallel for #endif for (w = 0; w < numWorkers; w++) { @@ -124,7 +124,7 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void while (!tif) { tif = TIFFOpen(fileName, "r"); if (outCounter == 3) { -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp critical #endif { @@ -146,7 +146,7 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void while (!TIFFSetDirectory(tif, (uint64_t)dir) && counter < 3) { counter++; if (counter == 3) { -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp critical #endif { @@ -162,7 +162,7 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void // loading the data into a buffer int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize * x * bytes); if (cBytes < 0) { -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp critical #endif { @@ -274,7 +274,7 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void uint64_t size = x * y * z * (bits / 8); void * tiffC = malloc(size); memcpy(tiffC, tiff, size); -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp parallel for #endif for (uint64_t k = 0; k < z; k++) { @@ -321,7 +321,7 @@ readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, uint64_t bytes = bits / 8; int32_t w; -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp parallel for #endif for (w = 0; w < numWorkers; w++) { @@ -402,7 +402,7 @@ readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, voi uint8_t errBak = 0; char errString[10000]; -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp parallel for #endif for (w = 0; w < numWorkers; w++) { @@ -412,7 +412,7 @@ readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, voi while (!tif) { tif = TIFFOpen(fileName, "r"); if (outCounter == 3) { -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp critical #endif { @@ -432,7 +432,7 @@ readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, voi counter + 1); counter++; if (counter == 3) { -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp critical #endif { @@ -447,7 +447,7 @@ readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, voi // loading the data into a buffer int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize * x * bytes); if (cBytes < 0) { -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp critical #endif { @@ -569,7 +569,7 @@ readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char *fileName, // Swap endianess for types greater than 8 bits // TODO: May need to change later because we may not always need to swap if (bits > 8) { -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp parallel for #endif for (uint64_t i = 0; i < x * y * z; i++) { @@ -609,7 +609,7 @@ readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char *fileName, uint64_t size = x * y * z * (bits / 8); void * tiffC = malloc(size); memcpy(tiffC, tiff, size); -#ifdef USE_OMP +#ifdef ENABLE_OPENMP #pragma omp parallel for #endif for (uint64_t k = 0; k < z; k++) { diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 2a47c561a..565c20b0c 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -22,7 +22,6 @@ parse_console_args(int argc, char *argv[], char **file_name) int c, parse_code = -1; while ((c = getopt(argc, argv, "f:")) != -1) { - printf("c : %c \n", c); switch (c) { case 'f': *file_name = optarg; @@ -46,15 +45,16 @@ main(int argc, char *argv[]) int i = 0; char bytes[10]; char *tiff_str_ptr; - printf("Program started!\n"); + // parse console argument int parse_code = parse_console_args(argc, argv, &file_name); - printf("parse_code %d\n", parse_code); if (parse_code) { return parse_code; } + // print file name for validating purpose printf("Filename: %s\n", file_name ? file_name : "(none)"); + // calling tiff loading process. parallel_TIFF_load(file_name, &tiff, 1, NULL); tiff_str_ptr = (char *)tiff; From a3ab20fe168c84cd85ee72045b0a8ec6eefe1159 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 20 Apr 2023 00:15:21 -0500 Subject: [PATCH 113/216] update CMakeLists --- tools/llsm/parallelReadTiff.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 0a8f609fd..d9a2d54f0 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -1,7 +1,7 @@ #include "parallelReadTiff.h" #include "tiffio.h" -#define ENABLE_OPENMP +// #define ENABLE_OPENMP #ifdef ENABLE_OPENMP #include "omp.h" From ac06b1276d7dba18dee9f06bc5597e104b647365 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 20 Apr 2023 00:32:22 -0500 Subject: [PATCH 114/216] update CMakeLists --- tools/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index b6123afc3..394b8a085 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -123,7 +123,7 @@ if(USE_LIB_TIFF) target_link_libraries(llsm_importer cjson) target_link_libraries(llsm_importer ${TIFF_LIBRARIES}) target_link_libraries(llsm_importer llsm_tiff) - target_include_directories(llsm_tiff PUBLIC ${PDC_INCLUDE_DIR}) + target_include_directories(llsm_importer PUBLIC ${PDC_INCLUDE_DIR}) else() message(WARNING "LibTiff not found, ignore building the executables which requires LibTiff support.") endif() From 6e5a046801be09f6229cf72a0f67a2095d7f52aa Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 20 Apr 2023 14:35:25 -0500 Subject: [PATCH 115/216] update CMakeLists --- tools/CMakeLists.txt | 2 +- tools/llsm/imageListReader.c | 71 +++++++++++++++++++++++++++++++++++ tools/llsm/imageListReader.h | 28 ++++++++++++++ tools/llsm/parallelReadTiff.c | 15 ++++---- tools/llsm/parallelReadTiff.h | 4 +- tools/llsm_importer.c | 53 +++++++++++++++++++------- 6 files changed, 150 insertions(+), 23 deletions(-) create mode 100644 tools/llsm/imageListReader.c create mode 100644 tools/llsm/imageListReader.h diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 394b8a085..fed5d93d3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -110,7 +110,7 @@ if(USE_LIB_TIFF) if(TIFF_FOUND) # Add the LibTIFF include directory to the include path include_directories(${TIFF_INCLUDE_DIRS}) - add_library(llsm_tiff llsm/parallelReadTiff.c llsm/parallelReadTiff.h) + add_library(llsm_tiff llsm/parallelReadTiff.c llsm/imageListReader.c) target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS}) target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES}) target_link_libraries(llsm_tiff PUBLIC ${TIFF_LIBRARIES}) diff --git a/tools/llsm/imageListReader.c b/tools/llsm/imageListReader.c new file mode 100644 index 000000000..199ec788c --- /dev/null +++ b/tools/llsm/imageListReader.c @@ -0,0 +1,71 @@ +#include "imageListReader.h" +#include "parallelReadTiff.h" + +void +scan_image_list(char *imageListFileName, on_image_ptr_t image_callback) +{ + FILE* file = fopen(imageListFileName, "r"); + + if (file == NULL) { + printf("Error: could not open file %s\n", imageListFileName); + return; + } + + char buffer[1024]; + char* token; + int line_count = 0; + image_file_info_t image_info; + + // Read and discard the first line (the header) + fgets(buffer, sizeof(buffer), file); + line_count++; + + // Read the remaining lines of the file + while (fgets(buffer, sizeof(buffer), file)) { + line_count++; + + // Tokenize the line by comma + token = strtok(buffer, ","); + + // Extract the filepath + image_info.filepath = strdup(token); + + // Extract the filename + token = strtok(NULL, ","); + image_info.filename = strdup(token); + + // Extract the stageX_um_ + token = strtok(NULL, ","); + image_info.stageX_um_ = atof(token); + + // Extract the stageY_um_ + token = strtok(NULL, ","); + image_info.stageY_um_ = atof(token); + + // Extract the stageZ_um_ + token = strtok(NULL, ","); + image_info.stageZ_um_ = atof(token); + + // Extract the objectiveX_um_ + token = strtok(NULL, ","); + image_info.objectiveX_um_ = atof(token); + + // Extract the objectiveY_um_ + token = strtok(NULL, ","); + image_info.objectiveY_um_ = atof(token); + + // Extract the objectiveZ_um_ + token = strtok(NULL, ","); + image_info.objectiveZ_um_ = atof(token); + + // Do something with the extracted image info... + if (image_callback != NULL) { + image_callback(&image_info); + } + + } + + fclose(file); + + printf("Read %d lines from file %s\n", line_count, imageListFileName); +} \ No newline at end of file diff --git a/tools/llsm/imageListReader.h b/tools/llsm/imageListReader.h new file mode 100644 index 000000000..db70a14fb --- /dev/null +++ b/tools/llsm/imageListReader.h @@ -0,0 +1,28 @@ +#ifndef IMAGELISTREADER_H +#define IMAGELISTREADER_H + +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + char *filepath; + char *filename; + double stageX_um_; + double stageY_um_; + double stageZ_um_; + double objectiveX_um_; + double objectiveY_um_; + double objectiveZ_um_; +} image_file_info_t; + +typedef void (*on_image_ptr_t)(image_file_info_t *); + +void scan_image_list(char *imageListFileName, on_image_ptr_t image_callback); + +#endif // IMAGELISTREADER_H \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index d9a2d54f0..8ad47b52d 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -7,13 +7,14 @@ #include "omp.h" #endif -#define CREATE_ARRAY(result_var, type, ndim, dim) do { \ - size_t i = 0, dim_prod = 1; \ - for (i = 0; i < (ndim); i++) { \ - dim_prod *= (dim)[i]; \ - } \ - result_var = (void *)malloc(dim_prod * sizeof(type)); \ -} while (0) +#define CREATE_ARRAY(result_var, type, ndim, dim) \ + do { \ + size_t i = 0, dim_prod = 1; \ + for (i = 0; i < (ndim); i++) { \ + dim_prod *= (dim)[i]; \ + } \ + result_var = (void *)malloc(dim_prod * sizeof(type)); \ + } while (0) void DummyHandler(const char *module, const char *fmt, va_list ap) diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h index 4f1d7ad5e..aec1dfe7b 100644 --- a/tools/llsm/parallelReadTiff.h +++ b/tools/llsm/parallelReadTiff.h @@ -11,8 +11,8 @@ #include typedef struct { - uint64_t *range; - size_t length; + uint64_t *range; + size_t length; } parallel_tiff_range_t; void parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tiff_range_t *strip_range); diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 565c20b0c..275f6fd9c 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -15,12 +15,13 @@ // #include "pdc_client_connect.h" #include "llsm/parallelReadTiff.h" +#include "llsm/imageListReader.h" int parse_console_args(int argc, char *argv[], char **file_name) { int c, parse_code = -1; - + while ((c = getopt(argc, argv, "f:")) != -1) { switch (c) { case 'f': @@ -36,15 +37,48 @@ parse_console_args(int argc, char *argv[], char **file_name) return parse_code; } +void +print_image_file_info(const image_file_info_t *image_info) +{ + printf("Filepath: %s\n", image_info->filepath); + printf("Filename: %s\n", image_info->filename); + printf("Stage X (um): %.2f\n", image_info->stageX_um_); + printf("Stage Y (um): %.2f\n", image_info->stageY_um_); + printf("Stage Z (um): %.2f\n", image_info->stageZ_um_); + printf("Objective X (um): %.2f\n", image_info->objectiveX_um_); + printf("Objective Y (um): %.2f\n", image_info->objectiveY_um_); + printf("Objective Z (um): %.2f\n", image_info->objectiveZ_um_); +} + +void +on_image(image_file_info_t *imageinfo) +{ + print_image_file_info(imageinfo); + // calling tiff loading process. + void *tiff = NULL; + int i = 0; + parallel_TIFF_load(imageinfo->filename, &tiff, 1, NULL); + + if (!tiff) + return 1; + + printf("first few bytes "); + for (i = 0; i < 10; i++) { + printf("%d ", ((uint8_t *)tiff)[i]); + } + printf("\n"); + free(tiff); +} + int main(int argc, char *argv[]) { char *file_name = NULL; - void *tiff = NULL; - int i = 0; - char bytes[10]; - char *tiff_str_ptr; + + + char bytes[10]; + // parse console argument int parse_code = parse_console_args(argc, argv, &file_name); if (parse_code) { @@ -54,14 +88,7 @@ main(int argc, char *argv[]) // print file name for validating purpose printf("Filename: %s\n", file_name ? file_name : "(none)"); - // calling tiff loading process. - parallel_TIFF_load(file_name, &tiff, 1, NULL); - - tiff_str_ptr = (char *)tiff; - for (i = 0; i < 10; i++) { - bytes[i] = tiff_str_ptr[i]; - } - printf("first few bytes : %s\n", bytes); + scan_image_list(file_name, &on_image); return 0; } \ No newline at end of file From 65c96d9063556002c635ce3dd362cc55fddb4b89 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 20 Apr 2023 15:31:14 -0500 Subject: [PATCH 116/216] update CMakeLists --- tools/llsm/imageListReader.c | 4 ++-- tools/llsm/imageListReader.h | 9 +++++++-- tools/llsm_importer.c | 30 ++++++++++++++++++++++-------- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/tools/llsm/imageListReader.c b/tools/llsm/imageListReader.c index 199ec788c..c9b0384b9 100644 --- a/tools/llsm/imageListReader.c +++ b/tools/llsm/imageListReader.c @@ -2,7 +2,7 @@ #include "parallelReadTiff.h" void -scan_image_list(char *imageListFileName, on_image_ptr_t image_callback) +scan_image_list(char *imageListFileName, on_image_ptr_t image_callback, img_scan_callback_args_t *args) { FILE* file = fopen(imageListFileName, "r"); @@ -60,7 +60,7 @@ scan_image_list(char *imageListFileName, on_image_ptr_t image_callback) // Do something with the extracted image info... if (image_callback != NULL) { - image_callback(&image_info); + image_callback(&image_info, args); } } diff --git a/tools/llsm/imageListReader.h b/tools/llsm/imageListReader.h index db70a14fb..fd3f646d9 100644 --- a/tools/llsm/imageListReader.h +++ b/tools/llsm/imageListReader.h @@ -21,8 +21,13 @@ typedef struct { double objectiveZ_um_; } image_file_info_t; -typedef void (*on_image_ptr_t)(image_file_info_t *); +typedef struct { + void *input; + void *output; +} img_scan_callback_args_t; + +typedef void (*on_image_ptr_t)(image_file_info_t *, img_scan_callback_args_t *args); -void scan_image_list(char *imageListFileName, on_image_ptr_t image_callback); +void scan_image_list(char *imageListFileName, on_image_ptr_t image_callback, img_scan_callback_args_t *args); #endif // IMAGELISTREADER_H \ No newline at end of file diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 275f6fd9c..4c893b011 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -16,6 +16,7 @@ #include "llsm/parallelReadTiff.h" #include "llsm/imageListReader.h" +#include int parse_console_args(int argc, char *argv[], char **file_name) @@ -51,13 +52,26 @@ print_image_file_info(const image_file_info_t *image_info) } void -on_image(image_file_info_t *imageinfo) +on_image(image_file_info_t *image_info, img_scan_callback_args_t *args) { - print_image_file_info(imageinfo); + print_image_file_info(image_info); + + char *dirname = (char *)args->input; + char filepath[256]; // calling tiff loading process. void *tiff = NULL; int i = 0; - parallel_TIFF_load(imageinfo->filename, &tiff, 1, NULL); + + // check if the path ends with a forward slash + if (dirname[strlen(dirname) - 1] != '/') { + strcat(dirname, "/"); // add a forward slash to the end of the path + } + + strcpy(filepath, dirname); // copy the directory path to the file path + strcat(filepath, image_info->filename); // concatenate the file name to the file path + + + parallel_TIFF_load(filepath, &tiff, 1, NULL); if (!tiff) return 1; @@ -75,20 +89,20 @@ main(int argc, char *argv[]) { char *file_name = NULL; - - - char bytes[10]; - + img_scan_callback_args_t callback_args; // parse console argument int parse_code = parse_console_args(argc, argv, &file_name); if (parse_code) { return parse_code; } + char* directory_path = dirname(strdup(directory_path)); // print file name for validating purpose printf("Filename: %s\n", file_name ? file_name : "(none)"); + printf("Directory: %s\n", directory_path ? directory_path : "(none)"); - scan_image_list(file_name, &on_image); + callback_args.input = (void *)directory_path; + scan_image_list(file_name, &on_image, &callback_args); return 0; } \ No newline at end of file From ec20bc7d9aaa1834af54dbd9b450544b983c8f52 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 20 Apr 2023 23:31:03 -0500 Subject: [PATCH 117/216] update CMakeLists --- tools/llsm_importer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 4c893b011..a37ad2853 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -95,7 +95,7 @@ main(int argc, char *argv[]) if (parse_code) { return parse_code; } - char* directory_path = dirname(strdup(directory_path)); + char* directory_path = dirname(strdup(file_name)); // print file name for validating purpose printf("Filename: %s\n", file_name ? file_name : "(none)"); From 46258f1648a4318119d26243c9678a3f1b60faeb Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 20 Apr 2023 23:32:10 -0500 Subject: [PATCH 118/216] fix return type --- tools/llsm_importer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index a37ad2853..7ab7d13ee 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -74,7 +74,7 @@ on_image(image_file_info_t *image_info, img_scan_callback_args_t *args) parallel_TIFF_load(filepath, &tiff, 1, NULL); if (!tiff) - return 1; + return; printf("first few bytes "); for (i = 0; i < 10; i++) { From 6a72a8d49ceec3eed41772834e5615b9449ca411 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 20 Apr 2023 23:35:48 -0500 Subject: [PATCH 119/216] fix return type --- tools/llsm/parallelReadTiff.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 8ad47b52d..4d6873ca8 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -39,7 +39,7 @@ readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, vo TIFF *tif = TIFFOpen(fileName, "r"); if (!tif) - printf("tiff:threadError", "Thread %d: File \"%s\" cannot be opened\n", w, fileName); + printf("tiff:threadError | Thread %d: File \"%s\" cannot be opened\n", w, fileName); void *buffer = malloc(x * bytes); for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); dir++) { @@ -241,17 +241,17 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void int fd = open(fileName, O_RDONLY); #endif if (fd == -1) - printf("disk:threadError", "File \"%s\" cannot be opened from Disk\n", fileName); + printf("disk:threadError | File \"%s\" cannot be opened from Disk\n", fileName); if (!tif) - printf("tiff:threadError", "File \"%s\" cannot be opened\n", fileName); + printf("tiff:threadError | File \"%s\" cannot be opened\n", fileName); uint64_t offset = 0; uint64_t *offsets = NULL; TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); uint64_t *byteCounts = NULL; TIFFGetField(tif, TIFFTAG_STRIPBYTECOUNTS, &byteCounts); if (!offsets || !byteCounts) - printf("tiff:threadError", "Could not get offsets or byte counts from the tiff file\n"); + printf("tiff:threadError | Could not get offsets or byte counts from the tiff file\n"); offset = offsets[0]; uint64_t fOffset = offsets[stripsPerDir - 1] + byteCounts[stripsPerDir - 1]; uint64_t zSize = fOffset - offset; @@ -308,7 +308,7 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void if (errBak) readTiffParallelBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); else - printf("tiff:threadError", errString); + printf("tiff:threadError %s\n", errString); } } @@ -329,7 +329,7 @@ readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, TIFF *tif = TIFFOpen(fileName, "r"); if (!tif) - printf("tiff:threadError", "Thread %d: File \"%s\" cannot be opened\n", w, fileName); + printf("tiff:threadError | Thread %d: File \"%s\" cannot be opened\n", w, fileName); void *buffer = malloc(x * bytes); for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); dir++) { @@ -517,7 +517,7 @@ readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, voi if (errBak) readTiffParallel2DBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); else - printf("tiff:threadError", errString); + printf("tiff:threadError %s\n", errString); } } @@ -533,7 +533,7 @@ readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char *fileName, #endif TIFF *tif = TIFFOpen(fileName, "r"); if (!tif) - printf("tiff:threadError", "File \"%s\" cannot be opened\n", fileName); + printf("tiff:threadError | File \"%s\" cannot be opened\n", fileName); uint64_t offset = 0; uint64_t *offsets = NULL; TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); @@ -682,7 +682,7 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u TIFFSetWarningHandler(DummyHandler); TIFF *tif = TIFFOpen(fileName, "r"); if (!tif) - printf("tiff:inputError", "File \"%s\" cannot be opened", fileName); + printf("tiff:inputError | File \"%s\" cannot be opened", fileName); TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, x); TIFFGetField(tif, TIFFTAG_IMAGELENGTH, y); @@ -714,13 +714,13 @@ get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, u } else { if (strip_range->length != 2) { - printf("tiff:inputError", "Input range is not 2"); + printf("tiff:inputError | Input range is not 2"); } else { *startSlice = (uint64_t)(*(strip_range->range)) - 1; *z = (uint64_t)(*(strip_range->range + 1)) - startSlice[0]; if (!TIFFSetDirectory(tif, startSlice[0] + z[0] - 1) || !TIFFSetDirectory(tif, startSlice[0])) { - printf("tiff:rangeOutOfBound", "Range is out of bounds"); + printf("tiff:rangeOutOfBound | Range is out of bounds"); } } } From 9e552574f82eee65b1b8b85864bedf9ee21f6a49 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 20 Apr 2023 23:58:08 -0500 Subject: [PATCH 120/216] add timing --- tools/llsm_importer.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 7ab7d13ee..bf86f6727 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -55,24 +55,34 @@ void on_image(image_file_info_t *image_info, img_scan_callback_args_t *args) { print_image_file_info(image_info); - + char *dirname = (char *)args->input; - char filepath[256]; + char filepath[256]; // calling tiff loading process. - void *tiff = NULL; - int i = 0; + void * tiff = NULL; + int i = 0; + struct timespec start, end; + double duration; // check if the path ends with a forward slash if (dirname[strlen(dirname) - 1] != '/') { strcat(dirname, "/"); // add a forward slash to the end of the path } - strcpy(filepath, dirname); // copy the directory path to the file path + strcpy(filepath, dirname); // copy the directory path to the file path strcat(filepath, image_info->filename); // concatenate the file name to the file path + clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation parallel_TIFF_load(filepath, &tiff, 1, NULL); + clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation + + duration = (end.tv_sec - start.tv_sec) * 1e9 + + (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds + + printf("Read %s Done! Time taken: %.4f seconds\n", filepath, duration / 1e9); + if (!tiff) return; @@ -88,14 +98,14 @@ int main(int argc, char *argv[]) { - char *file_name = NULL; + char * file_name = NULL; img_scan_callback_args_t callback_args; // parse console argument int parse_code = parse_console_args(argc, argv, &file_name); if (parse_code) { return parse_code; } - char* directory_path = dirname(strdup(file_name)); + char *directory_path = dirname(strdup(file_name)); // print file name for validating purpose printf("Filename: %s\n", file_name ? file_name : "(none)"); From 390be8a61d34e987843047e28cc546478d15f381 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 21 Apr 2023 12:52:40 -0500 Subject: [PATCH 121/216] add timing --- tools/llsm_importer.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index bf86f6727..322f1504e 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -18,6 +18,8 @@ #include "llsm/imageListReader.h" #include +int rank = 0, size = 1; + int parse_console_args(int argc, char *argv[], char **file_name) { @@ -51,6 +53,17 @@ print_image_file_info(const image_file_info_t *image_info) printf("Objective Z (um): %.2f\n", image_info->objectiveZ_um_); } +// void +// import_to_pdc(const image_file_info_t *image_info) { + + +// PDCprop_set_obj_dims(obj_prop_g, cur_ndims, (uint64_t *)cur_dims); +// PDCprop_set_obj_type(obj_prop_g, cur_type); +// PDCprop_set_obj_time_step(obj_prop_g, 0); +// PDCprop_set_obj_user_id(obj_prop_g, getuid()); +// PDCprop_set_obj_app_name(obj_prop_g, app_name); +// } + void on_image(image_file_info_t *image_info, img_scan_callback_args_t *args) { @@ -98,6 +111,12 @@ int main(int argc, char *argv[]) { +// #ifdef ENABLE_MPI +// MPI_Init(&argc, &argv); +// MPI_Comm_rank(MPI_COMM_WORLD, &rank); +// MPI_Comm_size(MPI_COMM_WORLD, &size); +// #endif + char * file_name = NULL; img_scan_callback_args_t callback_args; // parse console argument @@ -114,5 +133,10 @@ main(int argc, char *argv[]) callback_args.input = (void *)directory_path; scan_image_list(file_name, &on_image, &callback_args); + +// #ifdef ENABLE_MPI +// MPI_Finalize(); +// #endif + return 0; } \ No newline at end of file From a647af9c0b64e95a32c8a6318dd8503ea4d494be Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 21 Apr 2023 13:00:57 -0500 Subject: [PATCH 122/216] fix output --- tools/llsm/imageListReader.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/llsm/imageListReader.c b/tools/llsm/imageListReader.c index c9b0384b9..cf63891e7 100644 --- a/tools/llsm/imageListReader.c +++ b/tools/llsm/imageListReader.c @@ -67,5 +67,5 @@ scan_image_list(char *imageListFileName, on_image_ptr_t image_callback, img_scan fclose(file); - printf("Read %d lines from file %s\n", line_count, imageListFileName); + printf("Read %d lines from file %s after the header line.\n", line_count - 1, imageListFileName); } \ No newline at end of file From 5c0c37dce9131d05b05444b59cfe2421a347c63c Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 17:16:16 -0500 Subject: [PATCH 123/216] llsm tiff importer 1st version: read csv and import tiff files to PDC, adding metadata available in CSV files and TIFF loader --- tools/llsm/csvReader.c | 377 ++++++++++++++++++++++++++++++++++ tools/llsm/csvReader.h | 166 +++++++++++++++ tools/llsm/imageListReader.c | 71 ------- tools/llsm/imageListReader.h | 33 --- tools/llsm/parallelReadTiff.c | 15 +- tools/llsm/parallelReadTiff.h | 16 +- tools/llsm/pdc_list.c | 142 +++++++++++++ tools/llsm/pdc_list.h | 111 ++++++++++ tools/llsm_importer.c | 254 ++++++++++++++++++----- 9 files changed, 1031 insertions(+), 154 deletions(-) create mode 100644 tools/llsm/csvReader.c create mode 100644 tools/llsm/csvReader.h delete mode 100644 tools/llsm/imageListReader.c delete mode 100644 tools/llsm/imageListReader.h create mode 100644 tools/llsm/pdc_list.c create mode 100644 tools/llsm/pdc_list.h diff --git a/tools/llsm/csvReader.c b/tools/llsm/csvReader.c new file mode 100644 index 000000000..baeaf6684 --- /dev/null +++ b/tools/llsm/csvReader.c @@ -0,0 +1,377 @@ +#include "csvReader.h" + +csv_header_t * +csv_parse_header(char *line, char *field_types) +{ + csv_header_t *first_header = NULL; + csv_header_t *last_header = NULL; + char * token = NULL; + char * saveptr = NULL; + int field_index = 0; + int in_quotes = 0; + int value_start = 0; + int i = 0; + + for (int i = 0; line[i] != '\0'; ++i) { + if (line[i] == '\"') { + in_quotes = !in_quotes; + } + else if (!in_quotes && (line[i] == ',' || line[i + 1] == '\0')) { + // Allocate memory for the header struct + csv_header_t *header = (csv_header_t *)malloc(sizeof(csv_header_t)); + if (header == NULL) { + return NULL; + } + // Remove quotes and spaces from the field name + header->field_name = strndup(line + value_start, i - value_start + (line[i + 1] == '\0')); + + // Set the field index + header->field_index = field_index; + + // Set the field type + if (field_types != NULL) { + header->field_type = field_types[field_index]; + } + else { + header->field_type = 's'; + } + + // Set the next pointer to NULL + header->next = NULL; + + // Add the header to the linked list + if (first_header == NULL) { + first_header = header; + last_header = header; + } + else { + last_header->next = header; + last_header = header; + } + + value_start = i + 1; + field_index++; + } + } + + return first_header; +} + +csv_cell_t * +csv_parse_row(char *line, csv_header_t *header) +{ + csv_cell_t * first_cell = NULL; + csv_cell_t * last_cell = NULL; + csv_header_t *current_header = header; + char * token = NULL; + char * saveptr = NULL; + int field_index = 0; + int in_quotes = 0; + int value_start = 0; + int i = 0; + + for (int i = 0; line[i] != '\0'; ++i) { + if (line[i] == '\"') { + in_quotes = !in_quotes; + } + else if (!in_quotes && (line[i] == ',' || line[i + 1] == '\0')) { + // Allocate memory for the cell struct + csv_cell_t *cell = (csv_cell_t *)malloc(sizeof(csv_cell_t)); + if (cell == NULL) { + return NULL; + } + + // Set the field name + cell->header = current_header; + + // Set the field value + cell->field_value = strndup(line + value_start, i - value_start + (line[i + 1] == '\0')); + + // Set the next pointer to NULL + cell->next = NULL; + + // Add the cell to the linked list + if (first_cell == NULL) { + first_cell = cell; + last_cell = cell; + } + else { + last_cell->next = cell; + last_cell = cell; + } + + value_start = i + 1; + field_index++; + current_header = current_header->next; + } + } + + return first_cell; +} + +csv_cell_t * +csv_get_field_value_by_name(char *line, csv_header_t *header, char *field_name) +{ + csv_cell_t *cell = csv_parse_row(line, header); + while (cell != NULL) { + if (strcmp(cell->header->field_name, field_name) == 0) { + return cell; + } + cell = cell->next; + } + return NULL; +} + +csv_cell_t * +csv_get_field_value_by_index(char *line, csv_header_t *header, int field_index) +{ + csv_cell_t *cell = csv_parse_row(line, header); + while (cell != NULL) { + if (cell->header->field_index == field_index) { + return cell; + } + cell = cell->next; + } + return NULL; +} + +csv_table_t * +csv_parse_file(char *file_name, char *field_types) +{ + FILE *fp = fopen(file_name, "r"); + if (fp == NULL) { + return NULL; + } + + // Allocate memory for the table struct + csv_table_t *table = (csv_table_t *)malloc(sizeof(csv_table_t)); + if (table == NULL) { + return NULL; + } + + // Read the first line of the file + char * line = NULL; + size_t len = 0; + ssize_t read = getline(&line, &len, fp); + + // Parse the header + table->first_header = csv_parse_header(line, field_types); + + // Parse the rows + csv_row_t *first_row = NULL; + csv_row_t *last_row = NULL; + while ((read = getline(&line, &len, fp)) != -1) { + // Allocate memory for the row struct + csv_row_t *row = (csv_row_t *)malloc(sizeof(csv_row_t)); + if (row == NULL) { + return NULL; + } + + // Parse the row + row->first_cell = csv_parse_row(line, table->first_header); + + // Set the next pointer to NULL + row->next = NULL; + + // Add the row to the linked list + if (first_row == NULL) { + first_row = row; + last_row = row; + } + else { + last_row->next = row; + last_row = row; + } + } + + table->first_row = first_row; + + return table; +} + +csv_table_t *csv_parse_list(PDC_LIST *list, char *field_types){ + csv_table_t *table = (csv_table_t *)malloc(sizeof(csv_table_t)); + if (table == NULL) { + return NULL; + } + int num_file_read = 0; + csv_row_t *first_row = NULL; + csv_row_t *last_row = NULL; + + PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); + while (pdc_list_iterator_has_next(iter)) { + char *line = (char *)pdc_list_iterator_next(iter); + if (num_file_read == 0) { + table->first_header = csv_parse_header(line, field_types); + } else { + // Allocate memory for the row struct + csv_row_t *row = (csv_row_t *)malloc(sizeof(csv_row_t)); + if (row == NULL) { + return NULL; + } + + // Parse the row + row->first_cell = csv_parse_row(line, table->first_header); + + // Set the next pointer to NULL + row->next = NULL; + + // Add the row to the linked list + if (first_row == NULL) { + first_row = row; + last_row = row; + } + else { + last_row->next = row; + last_row = row; + } + } + num_file_read++; + } + + table->first_row = first_row; + + return table; +} + +void +csv_free_header(csv_header_t *header) +{ + csv_header_t *current_header = header; + csv_header_t *next_header = NULL; + while (current_header != NULL) { + next_header = current_header->next; + free(current_header->field_name); + free(current_header); + current_header = next_header; + } +} + +void +csv_free_row(csv_row_t *row) +{ + csv_row_t *current_row = row; + csv_row_t *next_row = NULL; + while (current_row != NULL) { + next_row = current_row->next; + csv_free_cell(current_row->first_cell); + free(current_row); + current_row = next_row; + } +} + +void +csv_free_cell(csv_cell_t *cell) +{ + csv_cell_t *current_cell = cell; + csv_cell_t *next_cell = NULL; + while (current_cell != NULL) { + next_cell = current_cell->next; + free(current_cell->field_value); + free(current_cell); + current_cell = next_cell; + } +} + +void +csv_free_table(csv_table_t *table) +{ + csv_free_header(table->first_header); + csv_free_row(table->first_row); + free(table); +} + +void +csv_print_header(csv_header_t *header) +{ + csv_header_t *current_header = header; + while (current_header != NULL) { + printf("%s", current_header->field_name); + if (current_header->next != NULL) { + printf(", "); + } + current_header = current_header->next; + } + printf("\n"); +} + +void +csv_print_row(csv_row_t *row, int with_key) +{ + csv_cell_t *current_cell = row->first_cell; + while (current_cell != NULL) { + csv_print_cell(current_cell, with_key); + if (current_cell->next != NULL) { + printf(", "); + if (with_key) { + printf("\n"); + } + } + current_cell = current_cell->next; + } + printf("\n"); +} + +void +csv_print_cell(csv_cell_t *cell, int with_key) +{ + if (with_key) { + printf("%s: ", cell->header->field_name); + } + switch (cell->header->field_type) + { + case 'i': + printf("%ld", strtol(cell->field_value, NULL, 10)); + break; + + case 'f': + printf("%f", strtod(cell->field_value, NULL)); + break; + + case 's': + printf("%s", cell->field_value); + break; + + default: + printf("%s", cell->field_value); + break; + } + +} + + + +void +csv_print_table(csv_table_t *table) +{ + csv_print_header(table->first_header); + csv_row_t *current_row = table->first_row; + while (current_row != NULL) { + csv_print_row(current_row, 0); + current_row = current_row->next; + } +} + +int +csv_get_num_rows(csv_table_t *table) +{ + int num_rows = 0; + csv_row_t *current_row = table->first_row; + while (current_row != NULL) { + num_rows++; + current_row = current_row->next; + } + return num_rows; +} + +int +csv_get_num_fields(csv_table_t *table) +{ + int num_fields = 0; + csv_header_t *current_header = table->first_header; + while (current_header != NULL) { + num_fields++; + current_header = current_header->next; + } + return num_fields; +} \ No newline at end of file diff --git a/tools/llsm/csvReader.h b/tools/llsm/csvReader.h new file mode 100644 index 000000000..a3743b052 --- /dev/null +++ b/tools/llsm/csvReader.h @@ -0,0 +1,166 @@ +#ifndef CSVREADER_H +#define CSVREADER_H + +#include +#include +#include + +#include "pdc_list.h" + +typedef struct csv_header_t { + char * field_name; + int field_index; + char field_type; + struct csv_header_t *next; +} csv_header_t; + +typedef struct csv_cell_t { + char * field_value; + csv_header_t * header; + struct csv_cell_t *next; +} csv_cell_t; + +typedef struct csv_row_t { + csv_cell_t * first_cell; + struct csv_row_t *next; +} csv_row_t; + +typedef struct csv_table_t { + csv_header_t *first_header; + csv_row_t * first_row; +} csv_table_t; + +/** + * @brief This function parses a CSV header line and returns a linked list of csv_header_t structs. The header + * string may contain quotes and spaces + * @param line The CSV header line to parse. + * @param field_types A string of field types. The field types are 's' for string, 'i' for long integer, 'f' + * for float, and 'd' for double. If this is NULL, all fields are assumed to be strings. + * + * @return A pointer to the first csv_header_t struct in the linked list. + */ +csv_header_t *csv_parse_header(char *line, char *field_types); + +/** + * @brief This function parse a CSV row line and returns a linked list of csv_cell_t structs. The row string + * may contain quotes and spaces + * @param line The CSV row line to parse. + * @param header A pointer to the first csv_header_t struct in the linked list. + * + * @return A pointer to the first csv_cell_t struct in the linked list. The value in the csv_cell should be + * free of quotes or spaces. + */ +csv_cell_t *csv_parse_row(char *line, csv_header_t *header); + +/** + * @brief This function returns the string value of a field for a given row string. The row string may contain + * quotes and spaces + * @param line The CSV row line to parse. + * @param header A pointer to the first csv_header_t struct in the linked list. + * @param field_name The name of the field to get the value for. + * + * @return A pointer to the csv_cell struct of the field. The value in the csv_cell should be free of quotes + * or spaces. + */ +csv_cell_t *csv_get_field_value_by_name(char *line, csv_header_t *header, char *field_name); + +/** + * @brief This function returns the string value of a field for a given row string. The row string may contain + * quotes and spaces + * @param line The CSV row line to parse. + * @param header A pointer to the first csv_header_t struct in the linked list. + * @param field_index The index of the field to get the value for. + * + * @return A pointer to the csv_cell struct of the field. The value in the csv_cell should be free of quotes + * or spaces. + */ +csv_cell_t *csv_get_field_value_by_index(char *line, csv_header_t *header, int field_index); + +/** + * @brief This function parses a CSV file and returns a csv_table_t struct. + * @param file_name The name of the CSV file to parse. + * @param field_types A string of field types. The field types are 's' for string, 'i' for long integer, 'f' + * for float, and 'd' for double. If this is NULL, all fields are assumed to be strings. + * + * @return A pointer to the csv_table_t struct. + */ +csv_table_t *csv_parse_file(char *file_name, char *field_types); + +/** + * @brief This function parses a PDC_LIST of strings as a CSV file and returns a csv_table_t struct. + * @param list A PDC_LIST of strings to parse. + * @param field_types A string of field types. The field types are 's' for string, 'i' for long integer, 'f' + * for float, and 'd' for double. If this is NULL, all fields are assumed to be strings. + * + * @return A pointer to the csv_table_t struct. + */ +csv_table_t *csv_parse_list(PDC_LIST *list, char *field_types); + +/** + * @brief This function frees the memory allocated for a csv_table_t struct. + * @param table A pointer to the csv_table_t struct to free. + */ +void csv_free_table(csv_table_t *table); + +/** + * @brief This function frees the memory allocated for a csv_header_t struct. + * @param header A pointer to the csv_header_t struct to free. + */ +void csv_free_header(csv_header_t *header); + +/** + * @brief This function frees the memory allocated for a csv_row_t struct. + * @param row A pointer to the csv_row_t struct to free. + */ +void csv_free_row(csv_row_t *row); + +/** + * @brief This function frees the memory allocated for a csv_cell_t struct. + * @param cell A pointer to the csv_cell_t struct to free. + */ +void csv_free_cell(csv_cell_t *cell); + +/** + * @brief This function prints the contents of a csv_table_t struct. + * @param table A pointer to the csv_table_t struct to print. + */ +void csv_print_table(csv_table_t *table); + +/** + * @brief This function prints the contents of a csv_header_t struct. + * @param header A pointer to the csv_header_t struct to print. + */ +void csv_print_header(csv_header_t *header); + +/** + * @brief This function prints the contents of a csv_row_t struct. + * @param row A pointer to the csv_row_t struct to print. + * @param with_key A flag to indicate whether to print the key or not. + */ + +void csv_print_row(csv_row_t *row, int with_key); + +/** + * @brief This function prints the contents of a csv_cell_t struct. + * @param cell A pointer to the csv_cell_t struct to print. + * @param with_key A flag to indicate whether to print the key or not. + */ +void csv_print_cell(csv_cell_t *cell, int with_key); + +/** + * @brief This function returns the number of rows in a csv_table_t struct. + * @param table A pointer to the csv_table_t struct. + * + * @return The number of rows in the table. + */ +int csv_get_num_rows(csv_table_t *table); + +/** + * @brief This function returns the number of fields in a csv_table_t struct. + * @param table A pointer to the csv_table_t struct. + * + * @return The number of fields in the table. + */ +int csv_get_num_fields(csv_table_t *table); + +#endif // CSVREADER_H \ No newline at end of file diff --git a/tools/llsm/imageListReader.c b/tools/llsm/imageListReader.c deleted file mode 100644 index cf63891e7..000000000 --- a/tools/llsm/imageListReader.c +++ /dev/null @@ -1,71 +0,0 @@ -#include "imageListReader.h" -#include "parallelReadTiff.h" - -void -scan_image_list(char *imageListFileName, on_image_ptr_t image_callback, img_scan_callback_args_t *args) -{ - FILE* file = fopen(imageListFileName, "r"); - - if (file == NULL) { - printf("Error: could not open file %s\n", imageListFileName); - return; - } - - char buffer[1024]; - char* token; - int line_count = 0; - image_file_info_t image_info; - - // Read and discard the first line (the header) - fgets(buffer, sizeof(buffer), file); - line_count++; - - // Read the remaining lines of the file - while (fgets(buffer, sizeof(buffer), file)) { - line_count++; - - // Tokenize the line by comma - token = strtok(buffer, ","); - - // Extract the filepath - image_info.filepath = strdup(token); - - // Extract the filename - token = strtok(NULL, ","); - image_info.filename = strdup(token); - - // Extract the stageX_um_ - token = strtok(NULL, ","); - image_info.stageX_um_ = atof(token); - - // Extract the stageY_um_ - token = strtok(NULL, ","); - image_info.stageY_um_ = atof(token); - - // Extract the stageZ_um_ - token = strtok(NULL, ","); - image_info.stageZ_um_ = atof(token); - - // Extract the objectiveX_um_ - token = strtok(NULL, ","); - image_info.objectiveX_um_ = atof(token); - - // Extract the objectiveY_um_ - token = strtok(NULL, ","); - image_info.objectiveY_um_ = atof(token); - - // Extract the objectiveZ_um_ - token = strtok(NULL, ","); - image_info.objectiveZ_um_ = atof(token); - - // Do something with the extracted image info... - if (image_callback != NULL) { - image_callback(&image_info, args); - } - - } - - fclose(file); - - printf("Read %d lines from file %s after the header line.\n", line_count - 1, imageListFileName); -} \ No newline at end of file diff --git a/tools/llsm/imageListReader.h b/tools/llsm/imageListReader.h deleted file mode 100644 index fd3f646d9..000000000 --- a/tools/llsm/imageListReader.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef IMAGELISTREADER_H -#define IMAGELISTREADER_H - -#include -#include -#include -#include -#include -#include -#include -#include - -typedef struct { - char *filepath; - char *filename; - double stageX_um_; - double stageY_um_; - double stageZ_um_; - double objectiveX_um_; - double objectiveY_um_; - double objectiveZ_um_; -} image_file_info_t; - -typedef struct { - void *input; - void *output; -} img_scan_callback_args_t; - -typedef void (*on_image_ptr_t)(image_file_info_t *, img_scan_callback_args_t *args); - -void scan_image_list(char *imageListFileName, on_image_ptr_t image_callback, img_scan_callback_args_t *args); - -#endif // IMAGELISTREADER_H \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 4d6873ca8..f36c13ea3 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -781,7 +781,7 @@ _TIFF_load(char *fileName, uint8_t isImageJIm, uint64_t x, uint64_t y, uint64_t } void -parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tiff_range_t *strip_range) +parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, image_info_t **image_info) { uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 0, is_imageJ = 0, imageJ_Z = 0; @@ -793,5 +793,16 @@ parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tif dims[1] = flipXY ? x : y; dims[2] = z; - _TIFF_load(fileName, is_imageJ, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, tiff_ptr); + *image_info = (image_info_t *)malloc(sizeof(image_info_t)); + (*image_info)->x = dims[0]; + (*image_info)->y = dims[1]; + (*image_info)->z = dims[2]; + (*image_info)->bits = bits; + (*image_info)->startSlice = startSlice; + (*image_info)->stripeSize = stripeSize; + (*image_info)->is_imageJ = is_imageJ; + (*image_info)->imageJ_Z = imageJ_Z; + (*image_info)->tiff_size = dims[0] * dims[1] * dims[2] * (bits / 8); + + _TIFF_load(fileName, is_imageJ, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, (void **)&((*image_info)->data)); } \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h index aec1dfe7b..60f1cb9f4 100644 --- a/tools/llsm/parallelReadTiff.h +++ b/tools/llsm/parallelReadTiff.h @@ -15,6 +15,20 @@ typedef struct { size_t length; } parallel_tiff_range_t; -void parallel_TIFF_load(char *fileName, void **tiff_ptr, uint8_t flipXY, parallel_tiff_range_t *strip_range); +typedef struct { + uint64_t x; + uint64_t y; + uint64_t z; + uint64_t bits; + uint64_t startSlice; + uint64_t stripeSize; + uint64_t is_imageJ; + uint64_t imageJ_Z; + void * tiff_ptr; + size_t tiff_size; +} image_info_t; + + +void parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, image_info_t **image_info_t); #endif // PARALLELREADTIFF_H \ No newline at end of file diff --git a/tools/llsm/pdc_list.c b/tools/llsm/pdc_list.c new file mode 100644 index 000000000..e6ea059be --- /dev/null +++ b/tools/llsm/pdc_list.c @@ -0,0 +1,142 @@ +#include "pdc_list.h" + + +PDC_LIST *pdc_list_new(){ + return pdc_list_create(100, 2.0); +} + +PDC_LIST* pdc_list_create(size_t initial_capacity, double expansion_factor) { + // Allocate memory for the list struct. + PDC_LIST* list = (PDC_LIST*) malloc(sizeof(PDC_LIST)); + if (list == NULL) { + return NULL; + } + + // Allocate memory for the array of items. + list->items = (void**) malloc(initial_capacity * sizeof(void*)); + if (list->items == NULL) { + free(list); + return NULL; + } + + // Initialize the list fields. + list->item_count = 0; + list->capacity = initial_capacity; + list->expansion_factor = expansion_factor; + + return list; +} + +void pdc_list_destroy(PDC_LIST* list) { + if (list == NULL) { + return; + } + + // Free all allocated memory for each item. + for (size_t i = 0; i < list->item_count; i++) { + free(list->items[i]); + } + + // Free the array of items and the list struct. + free(list->items); + free(list); +} + +void pdc_list_add(PDC_LIST* list, void* item) { + if (list == NULL || item == NULL) { + return; + } + + // Expand the array of items if necessary. + if (list->item_count >= list->capacity) { + list->capacity *= list->expansion_factor; + list->items = (void**) realloc(list->items, list->capacity * sizeof(void*)); + if (list->items == NULL) { + return; + } + } + + // Add the new item to the end of the array. + list->items[list->item_count++] = item; +} + +void* pdc_list_get(PDC_LIST* list, size_t index) { + if (list == NULL || index >= list->item_count) { + return NULL; + } + + // Return a pointer to the item at the given index. + return list->items[index]; +} + +size_t pdc_list_size(PDC_LIST* list) { + if (list == NULL) { + return 0; + } + + // Return the number of items in the list. + return list->item_count; +} + +void pdc_list_set_expansion_factor(PDC_LIST* list, double expansion_factor) { + if (list == NULL) { + return; + } + + // Set the new expansion factor for the list. + list->expansion_factor = expansion_factor; +} + +double pdc_list_get_expansion_factor(PDC_LIST* list) { + if (list == NULL) { + return 0; + } + + // Return the current expansion factor for the list. + return list->expansion_factor; +} + +PDC_LIST_ITERATOR* pdc_list_iterator_new(PDC_LIST* list) { + if (list == NULL) { + return NULL; + } + + // Allocate memory for the iterator struct. + PDC_LIST_ITERATOR* iterator = (PDC_LIST_ITERATOR*) malloc(sizeof(PDC_LIST_ITERATOR)); + if (iterator == NULL) { + return NULL; + } + + // Initialize the iterator fields. + iterator->list = list; + iterator->index = 0; + + return iterator; +} + +void pdc_list_iterator_destroy(PDC_LIST_ITERATOR* iterator) { + if (iterator == NULL) { + return; + } + + // Free the iterator struct. + free(iterator); +} + +void* pdc_list_iterator_next(PDC_LIST_ITERATOR* iterator) { + if (iterator == NULL) { + return NULL; + } + + // Return the next item in the list. + return pdc_list_get(iterator->list, iterator->index++); +} + +int pdc_list_iterator_has_next(PDC_LIST_ITERATOR* iterator) { + if (iterator == NULL) { + return 0; + } + + // Return true if there are more items in the list. + return iterator->index < pdc_list_size(iterator->list); +} \ No newline at end of file diff --git a/tools/llsm/pdc_list.h b/tools/llsm/pdc_list.h new file mode 100644 index 000000000..9847469e5 --- /dev/null +++ b/tools/llsm/pdc_list.h @@ -0,0 +1,111 @@ +#ifndef PDC_LIST_H +#define PDC_LIST_H + +#include + +/** + * A generic list data structure that stores a variable number of items of any type. + */ +typedef struct { + void **items; // Pointer to the array of items. + size_t item_count; // Number of items in the list. + size_t capacity; // Capacity of the array of items. + double expansion_factor; // Factor by which the capacity is expanded. +} PDC_LIST; + +/** + * A generic iterator for iterating over the items in a PDC_LIST. + */ +typedef struct { + PDC_LIST *list; // The list being iterated over. + size_t index; // The index of the next item to be returned. +} PDC_LIST_ITERATOR; + + +/** + * Creates a new PDC_LIST with default initial capacity 100 and default expansion factor 2.0. + * @return A pointer to the new PDC_LIST. + */ +PDC_LIST *pdc_list_new(); + +/** + * Creates a new PDC_LIST with the given initial capacity and expansion factor. + * @param initial_capacity The initial capacity of the list. + * @param expansion_factor The factor by which the capacity is expanded when the list is full. + * + * @return A pointer to the new PDC_LIST. + */ +PDC_LIST *pdc_list_create(size_t initial_capacity, double expansion_factor); + +/** + * Destroys the given PDC_LIST and frees all allocated memory. + * @param list The PDC_LIST to destroy. + */ +void pdc_list_destroy(PDC_LIST *list); + +/** + * Adds the given item to the end of the given PDC_LIST. + * @param list The PDC_LIST to add the item to. + * @param item The item to add to the PDC_LIST. + * + */ +void pdc_list_add(PDC_LIST *list, void *item); + +/** + * Gets the item at the given index in the given PDC_LIST. + * @param list The PDC_LIST to get the item from. + * @param index The index of the item to get. + * + * @return A pointer to the item at the given index. + */ +void *pdc_list_get(PDC_LIST *list, size_t index); + +/** + * Sets the item at the given index in the given PDC_LIST. + * @param list The PDC_LIST to set the item in. + * + * @return The number of items in the list. + */ +size_t pdc_list_size(PDC_LIST *list); + +/** + * Sets the expansion factor for the given PDC_LIST. + * @param list The PDC_LIST to set the expansion factor for. + * @param expansion_factor The factor by which the capacity is expanded when the list is full. + */ +void pdc_list_set_expansion_factor(PDC_LIST *list, double expansion_factor); + +/** + * Gets the expansion factor for the given PDC_LIST. + * @param list The PDC_LIST to get the expansion factor for. + */ +double pdc_list_get_expansion_factor(PDC_LIST *list); + +/** + * Creates a new PDC_LIST_ITERATOR for the given PDC_LIST. + * @param list The PDC_LIST to create the iterator for. + * @return A pointer to the new PDC_LIST_ITERATOR. + */ +PDC_LIST_ITERATOR *pdc_list_iterator_new(PDC_LIST *list); + +/** + * Destroys the given PDC_LIST_ITERATOR and frees all allocated memory. + * @param iterator The PDC_LIST_ITERATOR to destroy. + */ +void pdc_list_iterator_destroy(PDC_LIST_ITERATOR *iterator); + +/** + * Returns the next item in the PDC_LIST_ITERATOR. + * @param iterator The PDC_LIST_ITERATOR to get the next item from. + * @return A pointer to the next item in the PDC_LIST_ITERATOR. + */ +void *pdc_list_iterator_next(PDC_LIST_ITERATOR *iterator); + +/** + * Returns true if the PDC_LIST_ITERATOR has more items. + * @param iterator The PDC_LIST_ITERATOR to check. + * @return True if the PDC_LIST_ITERATOR has more items. + */ +int pdc_list_iterator_has_next(PDC_LIST_ITERATOR *iterator); + +#endif // PDC_LIST_H \ No newline at end of file diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 322f1504e..ffc87282c 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -4,10 +4,10 @@ #include #include -// #define ENABLE_MPI 1 +#define ENABLE_MPI 1 #ifdef ENABLE_MPI -// #include "mpi.h" +#include "mpi.h" #endif #include "pdc.h" @@ -15,10 +15,19 @@ // #include "pdc_client_connect.h" #include "llsm/parallelReadTiff.h" -#include "llsm/imageListReader.h" +#include "llsm/pdc_list.h" +#include "llsm/csvReader.h" #include -int rank = 0, size = 1; +typedef struct llsm_importer_args_t { + char * directory_path; + csv_header_t *csv_header; +} llsm_importer_args_t; + +int rank = 0, size = 1; + +pdcid_t pdc, cont_prop, cont, obj_prop; +pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; int parse_console_args(int argc, char *argv[], char **file_name) @@ -41,84 +50,175 @@ parse_console_args(int argc, char *argv[], char **file_name) } void -print_image_file_info(const image_file_info_t *image_info) +import_to_pdc(const image_info_t *image_info, const csv_cell_t *fileName_cell) { - printf("Filepath: %s\n", image_info->filepath); - printf("Filename: %s\n", image_info->filename); - printf("Stage X (um): %.2f\n", image_info->stageX_um_); - printf("Stage Y (um): %.2f\n", image_info->stageY_um_); - printf("Stage Z (um): %.2f\n", image_info->stageZ_um_); - printf("Objective X (um): %.2f\n", image_info->objectiveX_um_); - printf("Objective Y (um): %.2f\n", image_info->objectiveY_um_); - printf("Objective Z (um): %.2f\n", image_info->objectiveZ_um_); -} + struct timespec start, end; + double duration; + + clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation + + pdcid_t cur_obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); + + psize_t ndims = 3; + // FIXME: we should support uint64_t. + uint64_t dims[3] = {image_info->x, image_info->y, image_info->z}; + // FIXME: we should change the ndims parameter to psize_t type. + PDCprop_set_obj_dims(obj_prop_g, (PDC_int_t)ndims, dims); + PDCprop_set_obj_type(obj_prop_g, PDC_FLOAT); + PDCprop_set_obj_time_step(obj_prop_g, 0); + PDCprop_set_obj_user_id(obj_prop_g, getuid()); + PDCprop_set_obj_app_name(obj_prop_g, "LLSM"); + + // create object + // FIXME: There are many attributes currently in one file name, + // and we should do some research to see what would be a good object name for each image. + pdcid_t cur_obj_g = PDCobj_create(cont_id_g, fileName_cell->field_value, cur_obj_prop_g); + + // write data to object + pdcid_t local_region = PDCregion_create(ndims, 0, image_info->tiff_size); + pdcid_t remote_region = PDCregion_create(ndims, 0, image_info->tiff_size); + pdcid_t transfer_request = PDCregion_transfer_create(image_info->tiff_ptr, PDC_WRITE, cur_obj_g, local_region, remote_region); + PDCregion_transfer_start(transfer_request); + PDCregion_transfer_wait(transfer_request); + + // add metadata tags based on the csv row + csv_cell_t *cell = fileName_cell; + while (cell != NULL) { + char *field_name = cell->header->field_name; + char data_type = cell->header->field_type; + char *field_value = cell->field_value; + switch(data_type) { + case 'i': + int value = atoi(field_value); + PDCobj_put_tag(cur_obj_g, field_name, &value, sizeof(int)); + break; + case 'f': + double value = atof(field_value); + PDCobj_put_tag(cur_obj_g, field_name, &value, sizeof(double)); + break; + case 's': + PDCobj_put_tag(cur_obj_g, field_name, field_value, sizeof(char) * strlen(field_value)); + break; + default: + break; + } + cell = cell->next; + } -// void -// import_to_pdc(const image_file_info_t *image_info) { + // add extra metadata tags based on the image_info struct + PDCobj_put_tag(cur_obj_g, "x", &image_info->x, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "y", &image_info->y, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "z", &image_info->z, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "bits", &image_info->bits, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "startSlice", &image_info->startSlice, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "stripeSize", &image_info->stripeSize, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "is_imageJ", &image_info->is_imageJ, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "imageJ_Z", &image_info->imageJ_Z, sizeof(uint64_t)); + // close object + PDCobj_close(cur_obj_g); -// PDCprop_set_obj_dims(obj_prop_g, cur_ndims, (uint64_t *)cur_dims); -// PDCprop_set_obj_type(obj_prop_g, cur_type); -// PDCprop_set_obj_time_step(obj_prop_g, 0); -// PDCprop_set_obj_user_id(obj_prop_g, getuid()); -// PDCprop_set_obj_app_name(obj_prop_g, app_name); -// } + // get timing + clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation + duration = (end.tv_sec - start.tv_sec) * 1e9 + + (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds + + printf("[Rank %4d]create object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, duration / 1e9); + +} void -on_image(image_file_info_t *image_info, img_scan_callback_args_t *args) +on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) { - print_image_file_info(image_info); + csv_print_row(row, 1); - char *dirname = (char *)args->input; + char *dirname = strdup(llsm_args->directory_path); char filepath[256]; // calling tiff loading process. - void * tiff = NULL; - int i = 0; + image_info_t * image_info = NULL; + int i = 0; struct timespec start, end; double duration; + // Filepath,Filename,StageX_um_,StageY_um_,StageZ_um_,ObjectiveX_um_,ObjectiveY_um_,ObjectiveZ_um_ + + // get the file name from the csv row + csv_cell_t *fileName_cell = csv_get_field_value_by_name(row, llsm_args->csv_header, "Filename"); // check if the path ends with a forward slash if (dirname[strlen(dirname) - 1] != '/') { strcat(dirname, "/"); // add a forward slash to the end of the path } - strcpy(filepath, dirname); // copy the directory path to the file path - strcat(filepath, image_info->filename); // concatenate the file name to the file path + strcpy(filepath, dirname); // copy the directory path to the file path + strcat(filepath, fileName_cell->field_value); // concatenate the file name to the file path clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation - parallel_TIFF_load(filepath, &tiff, 1, NULL); + parallel_TIFF_load(filepath, 1, NULL, &image_info); clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation duration = (end.tv_sec - start.tv_sec) * 1e9 + (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds - printf("Read %s Done! Time taken: %.4f seconds\n", filepath, duration / 1e9); + printf("[Rand %4d]Read %s Done! Time taken: %.4f seconds\n", rank, filepath, duration / 1e9); - if (!tiff) + if (image_info == NULL || image_info->tiff_ptr == NULL) { return; + } printf("first few bytes "); for (i = 0; i < 10; i++) { - printf("%d ", ((uint8_t *)tiff)[i]); + printf("%d ", ((uint8_t *)image_info->tiff_ptr)[i]); } printf("\n"); - free(tiff); + + // import the image to PDC + import_to_pdc(image_info, fileName_cell); + + // free the image info + free(image_info->tiff_ptr); + free(image_info); + free(dirname); + + // free the csv row + csv_free_row(row); +} + +void +read_txt(char *txtFileName, PDC_LIST *list) +{ + FILE *file = fopen(txtFileName, "r"); + + if (file == NULL) { + printf("Error: could not open file %s\n", txtFileName); + return; + } + char buffer[1024]; + // Read the lines of the file + while (fgets(buffer, sizeof(buffer), file)) { + pdc_list_add(list, strdup(buffer)); + } } int main(int argc, char *argv[]) { -// #ifdef ENABLE_MPI -// MPI_Init(&argc, &argv); -// MPI_Comm_rank(MPI_COMM_WORLD, &rank); -// MPI_Comm_size(MPI_COMM_WORLD, &size); -// #endif +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif - char * file_name = NULL; - img_scan_callback_args_t callback_args; + char * file_name = NULL; + PDC_LIST * list = pdc_list_new(); + char * csv_line = NULL; + int num_row_read = 0; + csv_header_t * csv_header = NULL; + csv_row_t * csv_row = NULL; + llsm_importer_args_t *llsm_args = NULL; + char * csv_field_types = {'s', 's', 'f', 'f', 'f', 'f', 'f', 'f'}; // parse console argument int parse_code = parse_console_args(argc, argv, &file_name); if (parse_code) { @@ -130,13 +230,73 @@ main(int argc, char *argv[]) printf("Filename: %s\n", file_name ? file_name : "(none)"); printf("Directory: %s\n", directory_path ? directory_path : "(none)"); - callback_args.input = (void *)directory_path; - scan_image_list(file_name, &on_image, &callback_args); + // create a pdc + pdc_id_g = PDCinit("pdc"); + // create a container property + cont_prop_g = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); -// #ifdef ENABLE_MPI -// MPI_Finalize(); -// #endif + // create a container + cont = PDCcont_create("c1", cont_prop); + if (cont <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); + + // Rank 0 reads the filename list and distribute data to other ranks + if (rank == 0) { + read_txt(file_name, list); +#ifdef ENABLE_MPI + // broadcast the number of lines + int num_lines = pdc_list_size(list); + MPI_Bcast(&num_lines, 1, MPI_INT, 0, MPI_COMM_WORLD); + // broadcast the file names + PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); + while (pdc_list_iterator_has_next(iter)) { + char *csv_line = (char *)pdc_list_iterator_next(iter); + MPI_Bcast(csv_line, 256, MPI_CHAR, 0, MPI_COMM_WORLD); + } +#endif + } + else { +#ifdef ENABLE_MPI + // other ranks receive the number of files + int num_lines; + MPI_Bcast(&num_lines, 1, MPI_INT, 0, MPI_COMM_WORLD); + // receive the file names + int i; + for (i = 0; i < num_lines; i++) { + csv_line = (char *)malloc(256 * sizeof(char)); + MPI_Bcast(csv_line, 256, MPI_CHAR, 0, MPI_COMM_WORLD); + pdc_list_add(list, csv_line); + } +#endif + } + // parse the csv + csv_table_t *csv_table = csv_parse_list(list, csv_field_types); + if (csv_table == NULL) { + printf("Fail to parse csv file @ line %d!\n", __LINE__); + return -1; + } + llsm_args = (llsm_importer_args_t *)malloc(sizeof(llsm_importer_args_t)); + llsm_args->directory_path = directory_path; + llsm_args->csv_header = csv_table->first_header; + + // go through the csv table + csv_row_t *current_row = csv_table->first_row; + while (current_row != NULL) { + if (num_row_read % size == rank) { + on_csv_row(current_row, llsm_args); + } + num_row_read++; + current_row = current_row->next; + } + + csv_free_table(csv_table); + +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif return 0; -} \ No newline at end of file +} From 5615b8f25ab7a60493e98a048144603eae4a0ee8 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 17:18:30 -0500 Subject: [PATCH 124/216] fix vairable name --- tools/llsm/parallelReadTiff.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index f36c13ea3..727ba02c1 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -804,5 +804,5 @@ parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_ (*image_info)->imageJ_Z = imageJ_Z; (*image_info)->tiff_size = dims[0] * dims[1] * dims[2] * (bits / 8); - _TIFF_load(fileName, is_imageJ, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, (void **)&((*image_info)->data)); + _TIFF_load(fileName, is_imageJ, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, (void **)&((*image_info)->tiff_ptr)); } \ No newline at end of file From 015906ee2e5e8ccdfb71c77b33d3f671441db403 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 17:24:06 -0500 Subject: [PATCH 125/216] fix cmake --- tools/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index fed5d93d3..b0ae54dd2 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -108,9 +108,12 @@ option(USE_LIB_TIFF "Enable LibTiff." ON) if(USE_LIB_TIFF) find_package(TIFF REQUIRED) if(TIFF_FOUND) + set(LLSM_LIB_SOURCE + llsm/*.c + ) # Add the LibTIFF include directory to the include path include_directories(${TIFF_INCLUDE_DIRS}) - add_library(llsm_tiff llsm/parallelReadTiff.c llsm/imageListReader.c) + add_library(llsm_tiff ${LLSM_LIB_SOURCE}) target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS}) target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES}) target_link_libraries(llsm_tiff PUBLIC ${TIFF_LIBRARIES}) From 090cff8469901fbdab47c68296ee2e13b1ed4c26 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 17:25:27 -0500 Subject: [PATCH 126/216] fix cmake --- tools/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index b0ae54dd2..1c6940ca9 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -109,7 +109,9 @@ if(USE_LIB_TIFF) find_package(TIFF REQUIRED) if(TIFF_FOUND) set(LLSM_LIB_SOURCE - llsm/*.c + llsm/parallelReadTiff.c + llsm/csvReader.c + llsm/pdc_list.c ) # Add the LibTIFF include directory to the include path include_directories(${TIFF_INCLUDE_DIRS}) From 4e8c2113cbc5a9211bef57a18114ab06fa20ddd1 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 17:39:03 -0500 Subject: [PATCH 127/216] fix cmake --- tools/llsm_importer.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index ffc87282c..0d09e959d 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -69,14 +69,21 @@ import_to_pdc(const image_info_t *image_info, const csv_cell_t *fileName_cell) PDCprop_set_obj_user_id(obj_prop_g, getuid()); PDCprop_set_obj_app_name(obj_prop_g, "LLSM"); + uint64_t *offsets = (uint64_t *)malloc(sizeof(uint64_t) * ndims); + uint64_t *num_bytes = (uint64_t *)malloc(sizeof(uint64_t) * ndims); + for (int i = 0; i < ndims; i++) { + offsets[i] = 0; + num_bytes[i] = dims[i] * image_info->bits/8; + } + // create object // FIXME: There are many attributes currently in one file name, // and we should do some research to see what would be a good object name for each image. pdcid_t cur_obj_g = PDCobj_create(cont_id_g, fileName_cell->field_value, cur_obj_prop_g); // write data to object - pdcid_t local_region = PDCregion_create(ndims, 0, image_info->tiff_size); - pdcid_t remote_region = PDCregion_create(ndims, 0, image_info->tiff_size); + pdcid_t local_region = PDCregion_create(ndims, offsets, num_bytes); + pdcid_t remote_region = PDCregion_create(ndims, offsets, num_bytes); pdcid_t transfer_request = PDCregion_transfer_create(image_info->tiff_ptr, PDC_WRITE, cur_obj_g, local_region, remote_region); PDCregion_transfer_start(transfer_request); PDCregion_transfer_wait(transfer_request); @@ -125,6 +132,14 @@ import_to_pdc(const image_info_t *image_info, const csv_cell_t *fileName_cell) printf("[Rank %4d]create object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, duration / 1e9); + // free memory + free(offsets); + free(num_bytes); + PDCregion_close(local_region); + PDCregion_close(remote_region); + PDCregion_transfer_close(transfer_request); + PDCprop_close(cur_obj_prop_g); + } void @@ -180,9 +195,6 @@ on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) free(image_info->tiff_ptr); free(image_info); free(dirname); - - // free the csv row - csv_free_row(row); } void From 247e8b215ce83652066f62c42e7344079964840a Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 17:45:30 -0500 Subject: [PATCH 128/216] fix cmake --- tools/llsm_importer.c | 75 ++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 0d09e959d..f7edabb0c 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -50,7 +50,7 @@ parse_console_args(int argc, char *argv[], char **file_name) } void -import_to_pdc(const image_info_t *image_info, const csv_cell_t *fileName_cell) +import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) { struct timespec start, end; double duration; @@ -59,8 +59,8 @@ import_to_pdc(const image_info_t *image_info, const csv_cell_t *fileName_cell) pdcid_t cur_obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); - psize_t ndims = 3; - // FIXME: we should support uint64_t. + psize_t ndims = 3; + // FIXME: we should support uint64_t. uint64_t dims[3] = {image_info->x, image_info->y, image_info->z}; // FIXME: we should change the ndims parameter to psize_t type. PDCprop_set_obj_dims(obj_prop_g, (PDC_int_t)ndims, dims); @@ -69,39 +69,40 @@ import_to_pdc(const image_info_t *image_info, const csv_cell_t *fileName_cell) PDCprop_set_obj_user_id(obj_prop_g, getuid()); PDCprop_set_obj_app_name(obj_prop_g, "LLSM"); - uint64_t *offsets = (uint64_t *)malloc(sizeof(uint64_t) * ndims); - uint64_t *num_bytes = (uint64_t *)malloc(sizeof(uint64_t) * ndims); + uint64_t *offsets = (uint64_t *)malloc(sizeof(uint64_t) * ndims); + uint64_t *num_bytes = (uint64_t *)malloc(sizeof(uint64_t) * ndims); for (int i = 0; i < ndims; i++) { - offsets[i] = 0; - num_bytes[i] = dims[i] * image_info->bits/8; + offsets[i] = 0; + num_bytes[i] = dims[i] * image_info->bits / 8; } // create object - // FIXME: There are many attributes currently in one file name, + // FIXME: There are many attributes currently in one file name, // and we should do some research to see what would be a good object name for each image. pdcid_t cur_obj_g = PDCobj_create(cont_id_g, fileName_cell->field_value, cur_obj_prop_g); // write data to object - pdcid_t local_region = PDCregion_create(ndims, offsets, num_bytes); - pdcid_t remote_region = PDCregion_create(ndims, offsets, num_bytes); - pdcid_t transfer_request = PDCregion_transfer_create(image_info->tiff_ptr, PDC_WRITE, cur_obj_g, local_region, remote_region); + pdcid_t local_region = PDCregion_create(ndims, offsets, num_bytes); + pdcid_t remote_region = PDCregion_create(ndims, offsets, num_bytes); + pdcid_t transfer_request = + PDCregion_transfer_create(image_info->tiff_ptr, PDC_WRITE, cur_obj_g, local_region, remote_region); PDCregion_transfer_start(transfer_request); PDCregion_transfer_wait(transfer_request); // add metadata tags based on the csv row csv_cell_t *cell = fileName_cell; while (cell != NULL) { - char *field_name = cell->header->field_name; - char data_type = cell->header->field_type; + char *field_name = cell->header->field_name; + char data_type = cell->header->field_type; char *field_value = cell->field_value; - switch(data_type) { + switch (data_type) { case 'i': - int value = atoi(field_value); - PDCobj_put_tag(cur_obj_g, field_name, &value, sizeof(int)); + int ivalue = atoi(field_value); + PDCobj_put_tag(cur_obj_g, field_name, &ivalue, sizeof(int)); break; case 'f': - double value = atof(field_value); - PDCobj_put_tag(cur_obj_g, field_name, &value, sizeof(double)); + double fvalue = atof(field_value); + PDCobj_put_tag(cur_obj_g, field_name, &fvalue, sizeof(double)); break; case 's': PDCobj_put_tag(cur_obj_g, field_name, field_value, sizeof(char) * strlen(field_value)); @@ -113,14 +114,14 @@ import_to_pdc(const image_info_t *image_info, const csv_cell_t *fileName_cell) } // add extra metadata tags based on the image_info struct - PDCobj_put_tag(cur_obj_g, "x", &image_info->x, sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "y", &image_info->y, sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "z", &image_info->z, sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "bits", &image_info->bits, sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "startSlice", &image_info->startSlice, sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "stripeSize", &image_info->stripeSize, sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "is_imageJ", &image_info->is_imageJ, sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "imageJ_Z", &image_info->imageJ_Z, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "x", &(image_info->x), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "y", &(image_info->y), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "z", &(image_info->z), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "bits", &(image_info->bits), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "startSlice", &(image_info->startSlice), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "stripeSize", &(image_info->stripeSize), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "is_imageJ", &(image_info->is_imageJ), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "imageJ_Z", &(image_info->imageJ_Z), sizeof(uint64_t)); // close object PDCobj_close(cur_obj_g); @@ -130,7 +131,8 @@ import_to_pdc(const image_info_t *image_info, const csv_cell_t *fileName_cell) duration = (end.tv_sec - start.tv_sec) * 1e9 + (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds - printf("[Rank %4d]create object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, duration / 1e9); + printf("[Rank %4d]create object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, + duration / 1e9); // free memory free(offsets); @@ -139,7 +141,6 @@ import_to_pdc(const image_info_t *image_info, const csv_cell_t *fileName_cell) PDCregion_close(remote_region); PDCregion_transfer_close(transfer_request); PDCprop_close(cur_obj_prop_g); - } void @@ -164,7 +165,7 @@ on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) strcat(dirname, "/"); // add a forward slash to the end of the path } - strcpy(filepath, dirname); // copy the directory path to the file path + strcpy(filepath, dirname); // copy the directory path to the file path strcat(filepath, fileName_cell->field_value); // concatenate the file name to the file path clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation @@ -223,14 +224,14 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &size); #endif - char * file_name = NULL; - PDC_LIST * list = pdc_list_new(); - char * csv_line = NULL; - int num_row_read = 0; - csv_header_t * csv_header = NULL; - csv_row_t * csv_row = NULL; - llsm_importer_args_t *llsm_args = NULL; - char * csv_field_types = {'s', 's', 'f', 'f', 'f', 'f', 'f', 'f'}; + char * file_name = NULL; + PDC_LIST * list = pdc_list_new(); + char * csv_line = NULL; + int num_row_read = 0; + csv_header_t * csv_header = NULL; + csv_row_t * csv_row = NULL; + llsm_importer_args_t *llsm_args = NULL; + char csv_field_types[] = {'s', 's', 'f', 'f', 'f', 'f', 'f', 'f'}; // parse console argument int parse_code = parse_console_args(argc, argv, &file_name); if (parse_code) { From faed7ca2d6c3d346fe5fcc4673da78002c1a4855 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 17:58:14 -0500 Subject: [PATCH 129/216] fix cmake --- tools/llsm/csvReader.c | 32 +++++++++++--------------------- tools/llsm/csvReader.h | 12 ++++++------ 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/tools/llsm/csvReader.c b/tools/llsm/csvReader.c index baeaf6684..19b3dee8f 100644 --- a/tools/llsm/csvReader.c +++ b/tools/llsm/csvReader.c @@ -57,7 +57,7 @@ csv_parse_header(char *line, char *field_types) return first_header; } -csv_cell_t * +csv_row_t * csv_parse_row(char *line, csv_header_t *header) { csv_cell_t * first_cell = NULL; @@ -105,14 +105,16 @@ csv_parse_row(char *line, csv_header_t *header) current_header = current_header->next; } } - - return first_cell; + csv_row_t *row = (csv_row_t *)malloc(sizeof(csv_row_t)); + row->first_cell = first_cell; + row->next = NULL; + return row; } csv_cell_t * -csv_get_field_value_by_name(char *line, csv_header_t *header, char *field_name) +csv_get_field_value_by_name(csv_row_t *row, csv_header_t *header, char *field_name) { - csv_cell_t *cell = csv_parse_row(line, header); + csv_cell_t *cell = row->first_cell; while (cell != NULL) { if (strcmp(cell->header->field_name, field_name) == 0) { return cell; @@ -123,9 +125,9 @@ csv_get_field_value_by_name(char *line, csv_header_t *header, char *field_name) } csv_cell_t * -csv_get_field_value_by_index(char *line, csv_header_t *header, int field_index) +csv_get_field_value_by_index(csv_row_t *row, csv_header_t *header, int field_index) { - csv_cell_t *cell = csv_parse_row(line, header); + csv_cell_t *cell = row->first_cell; while (cell != NULL) { if (cell->header->field_index == field_index) { return cell; @@ -162,17 +164,11 @@ csv_parse_file(char *file_name, char *field_types) csv_row_t *last_row = NULL; while ((read = getline(&line, &len, fp)) != -1) { // Allocate memory for the row struct - csv_row_t *row = (csv_row_t *)malloc(sizeof(csv_row_t)); + csv_row_t *row = csv_parse_row(line, table->first_header); if (row == NULL) { return NULL; } - // Parse the row - row->first_cell = csv_parse_row(line, table->first_header); - - // Set the next pointer to NULL - row->next = NULL; - // Add the row to the linked list if (first_row == NULL) { first_row = row; @@ -205,17 +201,11 @@ csv_table_t *csv_parse_list(PDC_LIST *list, char *field_types){ table->first_header = csv_parse_header(line, field_types); } else { // Allocate memory for the row struct - csv_row_t *row = (csv_row_t *)malloc(sizeof(csv_row_t)); + csv_row_t *row = csv_parse_row(line, table->first_header); if (row == NULL) { return NULL; } - // Parse the row - row->first_cell = csv_parse_row(line, table->first_header); - - // Set the next pointer to NULL - row->next = NULL; - // Add the row to the linked list if (first_row == NULL) { first_row = row; diff --git a/tools/llsm/csvReader.h b/tools/llsm/csvReader.h index a3743b052..e7599ecb2 100644 --- a/tools/llsm/csvReader.h +++ b/tools/llsm/csvReader.h @@ -47,34 +47,34 @@ csv_header_t *csv_parse_header(char *line, char *field_types); * @param line The CSV row line to parse. * @param header A pointer to the first csv_header_t struct in the linked list. * - * @return A pointer to the first csv_cell_t struct in the linked list. The value in the csv_cell should be + * @return A pointer to the csv_row_t struct. The value in the csv_cell should be * free of quotes or spaces. */ -csv_cell_t *csv_parse_row(char *line, csv_header_t *header); +csv_row_t *csv_parse_row(char *line, csv_header_t *header); /** * @brief This function returns the string value of a field for a given row string. The row string may contain * quotes and spaces - * @param line The CSV row line to parse. + * @param row The CSV row to look for. * @param header A pointer to the first csv_header_t struct in the linked list. * @param field_name The name of the field to get the value for. * * @return A pointer to the csv_cell struct of the field. The value in the csv_cell should be free of quotes * or spaces. */ -csv_cell_t *csv_get_field_value_by_name(char *line, csv_header_t *header, char *field_name); +csv_cell_t *csv_get_field_value_by_name(csv_row_t *row, csv_header_t *header, char *field_name); /** * @brief This function returns the string value of a field for a given row string. The row string may contain * quotes and spaces - * @param line The CSV row line to parse. + * @param row The CSV row to look for. * @param header A pointer to the first csv_header_t struct in the linked list. * @param field_index The index of the field to get the value for. * * @return A pointer to the csv_cell struct of the field. The value in the csv_cell should be free of quotes * or spaces. */ -csv_cell_t *csv_get_field_value_by_index(char *line, csv_header_t *header, int field_index); +csv_cell_t *csv_get_field_value_by_index(csv_row_t *row, csv_header_t *header, int field_index); /** * @brief This function parses a CSV file and returns a csv_table_t struct. From 208baed0ded22d0ef5224ba6cf61a768a30c9a4d Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 17:59:19 -0500 Subject: [PATCH 130/216] fix cmake --- tools/llsm_importer.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index f7edabb0c..2c841a362 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -4,7 +4,9 @@ #include #include -#define ENABLE_MPI 1 +#ifndef ENABLE_MPI +#define ENABLE_MPI +#endif #ifdef ENABLE_MPI #include "mpi.h" From 4bbf585ba1936066049f6a023a6d1707b2b6b980 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:10:38 -0500 Subject: [PATCH 131/216] add scripts --- scripts/llsm_importer/clean.sh | 8 +++ scripts/llsm_importer/gen_script.sh | 21 ++++++++ scripts/llsm_importer/submit.sh | 76 +++++++++++++++++++++++++++++ scripts/llsm_importer/template.sh | 65 ++++++++++++++++++++++++ 4 files changed, 170 insertions(+) create mode 100644 scripts/llsm_importer/clean.sh create mode 100644 scripts/llsm_importer/gen_script.sh create mode 100644 scripts/llsm_importer/submit.sh create mode 100644 scripts/llsm_importer/template.sh diff --git a/scripts/llsm_importer/clean.sh b/scripts/llsm_importer/clean.sh new file mode 100644 index 000000000..d6476962e --- /dev/null +++ b/scripts/llsm_importer/clean.sh @@ -0,0 +1,8 @@ +#!/bin/bash +MAX_NODE=512 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + + rm -rf $i/* + +done \ No newline at end of file diff --git a/scripts/llsm_importer/gen_script.sh b/scripts/llsm_importer/gen_script.sh new file mode 100644 index 000000000..9d310f2bb --- /dev/null +++ b/scripts/llsm_importer/gen_script.sh @@ -0,0 +1,21 @@ +#!/bin/bash +N_THREAD=NO +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +PROG_BASENAME=llsm_importer + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + mkdir -p $i + JOBNAME=${PROG_BASENAME}_${i} + TARGET=./$i/$JOBNAME.sbatch + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + if [[ "$i" -gt "16" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi +done diff --git a/scripts/llsm_importer/submit.sh b/scripts/llsm_importer/submit.sh new file mode 100644 index 000000000..b9019d149 --- /dev/null +++ b/scripts/llsm_importer/submit.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# MIN_PROC=4 +# MAX_PROC=128 +MIN_PROC=1 +MAX_PROC=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +PROG_BASENAME=llsm_importer + +curdir=$(pwd) + +first_submit=1 + +for (( i = 1; i <= $MAX_PROC; i*=2 )); do + mkdir -p $i + JOBNAME=${PROG_BASENAME}_${i} + TARGET=./$i/JOBNAME.sh + + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` + echo $njob + done + + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi + + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime +done + + +# for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do + +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# while [ $njob -ge 4 ] +# do +# sleeptime=$[ ( $RANDOM % 1000 ) ] +# sleep $sleeptime +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# done + + +# cd $curdir/$j +# for filename in ./*.sh ; do + +# if [[ $first_submit == 1 ]]; then +# # Submit first job w/o dependency +# echo "Submitting $filename" +# job=`sbatch $filename` +# first_submit=0 +# else +# echo "Submitting $filename after ${job: -8}" +# job=`sbatch -d afterany:${job: -8} $filename` +# fi + +# sleeptime=$[ ( $RANDOM % 5 ) ] +# sleep $sleeptime + +# done +# done diff --git a/scripts/llsm_importer/template.sh b/scripts/llsm_importer/template.sh new file mode 100644 index 000000000..c7074ec13 --- /dev/null +++ b/scripts/llsm_importer/template.sh @@ -0,0 +1,65 @@ +#!/bin/bash -l + +#REGSBATCH -q regular +#DBGSBATCH -q debug +#SBATCH -N NODENUM +#REGSBATCH -t 1:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH -C cpu +#SBATCH -J JOBNAME +#SBATCH -A m2621 +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + +# export PDC_DEBUG=0 + + +export PDC_TMPDIR=$SCRATCH/data/pdc/conf + +rm -rf $PDC_TMPDIR/* + +REPEAT=1 + +N_NODE=NODENUM +NCLIENT=31 +# NCLIENT=126 + +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE +mkdir -p $PDC_TMPDIR + +let TOTALPROC=$NCLIENT*$N_NODE + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin +TOOLPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/source/pdc_llsm/tools/build +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$TOOLPATH/llsm_importer +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* +chmod +x $TOOLPATH/llsm_importer + +IMGLIST_PATH=/global/cfs/cdirs/m2621/wzhang5/data/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1/ImageList_from_encoder.csv + +date + + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT -f $IMGLIST_PATH + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE + +date From 227e85a00fdb07e21eb819f88074dfa581253624 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:11:44 -0500 Subject: [PATCH 132/216] add scripts --- scripts/llsm_importer/clean.sh | 0 scripts/llsm_importer/gen_script.sh | 0 scripts/llsm_importer/submit.sh | 0 scripts/llsm_importer/template.sh | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/llsm_importer/clean.sh mode change 100644 => 100755 scripts/llsm_importer/gen_script.sh mode change 100644 => 100755 scripts/llsm_importer/submit.sh mode change 100644 => 100755 scripts/llsm_importer/template.sh diff --git a/scripts/llsm_importer/clean.sh b/scripts/llsm_importer/clean.sh old mode 100644 new mode 100755 diff --git a/scripts/llsm_importer/gen_script.sh b/scripts/llsm_importer/gen_script.sh old mode 100644 new mode 100755 diff --git a/scripts/llsm_importer/submit.sh b/scripts/llsm_importer/submit.sh old mode 100644 new mode 100755 diff --git a/scripts/llsm_importer/template.sh b/scripts/llsm_importer/template.sh old mode 100644 new mode 100755 From 421b00e69f2872a67c6d453d0f27857762661d63 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:13:35 -0500 Subject: [PATCH 133/216] add scripts --- scripts/llsm_importer/template.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/llsm_importer/template.sh b/scripts/llsm_importer/template.sh index c7074ec13..d9305c83e 100755 --- a/scripts/llsm_importer/template.sh +++ b/scripts/llsm_importer/template.sh @@ -21,7 +21,7 @@ rm -rf $PDC_TMPDIR/* REPEAT=1 N_NODE=NODENUM -NCLIENT=31 +NCLIENT=1 # NCLIENT=126 export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE From 8d0f24d29fdb2f46cfeeabe8d3b0431b7d272646 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:18:07 -0500 Subject: [PATCH 134/216] debugging for nonMPI program --- tools/llsm_importer.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 2c841a362..d052688af 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -9,7 +9,8 @@ #endif #ifdef ENABLE_MPI -#include "mpi.h" +// #include "mpi.h" +#undef ENABLE_MPI #endif #include "pdc.h" @@ -192,7 +193,7 @@ on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) printf("\n"); // import the image to PDC - import_to_pdc(image_info, fileName_cell); + // import_to_pdc(image_info, fileName_cell); // free the image info free(image_info->tiff_ptr); From 58de0a040ff756f02fc4cc7d6c600949fa094b81 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:20:40 -0500 Subject: [PATCH 135/216] debugging for nonMPI program --- tools/llsm_importer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index d052688af..17a40a2dd 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -246,6 +246,7 @@ main(int argc, char *argv[]) printf("Filename: %s\n", file_name ? file_name : "(none)"); printf("Directory: %s\n", directory_path ? directory_path : "(none)"); +#ifdef ENABLE_MPI // create a pdc pdc_id_g = PDCinit("pdc"); @@ -258,6 +259,7 @@ main(int argc, char *argv[]) cont = PDCcont_create("c1", cont_prop); if (cont <= 0) printf("Fail to create container @ line %d!\n", __LINE__); +#endif // Rank 0 reads the filename list and distribute data to other ranks if (rank == 0) { From 25c151397c1cebe560711c580956c97bf204a0a1 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:24:26 -0500 Subject: [PATCH 136/216] debugging for nonMPI program --- tools/llsm_importer.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 17a40a2dd..6ac2cfa2c 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -246,20 +246,20 @@ main(int argc, char *argv[]) printf("Filename: %s\n", file_name ? file_name : "(none)"); printf("Directory: %s\n", directory_path ? directory_path : "(none)"); -#ifdef ENABLE_MPI - // create a pdc - pdc_id_g = PDCinit("pdc"); - - // create a container property - cont_prop_g = PDCprop_create(PDC_CONT_CREATE, pdc); - if (cont_prop <= 0) - printf("Fail to create container property @ line %d!\n", __LINE__); - - // create a container - cont = PDCcont_create("c1", cont_prop); - if (cont <= 0) - printf("Fail to create container @ line %d!\n", __LINE__); -#endif + + // // create a pdc + // pdc_id_g = PDCinit("pdc"); + + // // create a container property + // cont_prop_g = PDCprop_create(PDC_CONT_CREATE, pdc); + // if (cont_prop <= 0) + // printf("Fail to create container property @ line %d!\n", __LINE__); + + // // create a container + // cont = PDCcont_create("c1", cont_prop); + // if (cont <= 0) + // printf("Fail to create container @ line %d!\n", __LINE__); + // Rank 0 reads the filename list and distribute data to other ranks if (rank == 0) { From d75bae273da785b8d3c2615a907653b2e4f6058b Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:30:28 -0500 Subject: [PATCH 137/216] clang format, without PDC, everything works perfectly. program fails at PDC init stage where PDCprop_create(PDC_CONT_CREATE, pdc) is being created --- tools/llsm/csvReader.c | 51 ++++++++++++++-------------- tools/llsm/csvReader.h | 2 +- tools/llsm/parallelReadTiff.c | 22 ++++++------ tools/llsm/parallelReadTiff.h | 4 +-- tools/llsm/pdc_list.c | 63 ++++++++++++++++++++++++----------- tools/llsm/pdc_list.h | 13 ++++---- tools/llsm_importer.c | 2 -- 7 files changed, 89 insertions(+), 68 deletions(-) diff --git a/tools/llsm/csvReader.c b/tools/llsm/csvReader.c index 19b3dee8f..39c4571aa 100644 --- a/tools/llsm/csvReader.c +++ b/tools/llsm/csvReader.c @@ -105,7 +105,7 @@ csv_parse_row(char *line, csv_header_t *header) current_header = current_header->next; } } - csv_row_t *row = (csv_row_t *)malloc(sizeof(csv_row_t)); + csv_row_t *row = (csv_row_t *)malloc(sizeof(csv_row_t)); row->first_cell = first_cell; row->next = NULL; return row; @@ -185,21 +185,24 @@ csv_parse_file(char *file_name, char *field_types) return table; } -csv_table_t *csv_parse_list(PDC_LIST *list, char *field_types){ +csv_table_t * +csv_parse_list(PDC_LIST *list, char *field_types) +{ csv_table_t *table = (csv_table_t *)malloc(sizeof(csv_table_t)); if (table == NULL) { return NULL; } - int num_file_read = 0; - csv_row_t *first_row = NULL; - csv_row_t *last_row = NULL; + int num_file_read = 0; + csv_row_t *first_row = NULL; + csv_row_t *last_row = NULL; PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); while (pdc_list_iterator_has_next(iter)) { char *line = (char *)pdc_list_iterator_next(iter); if (num_file_read == 0) { table->first_header = csv_parse_header(line, field_types); - } else { + } + else { // Allocate memory for the row struct csv_row_t *row = csv_parse_row(line, table->first_header); if (row == NULL) { @@ -308,29 +311,25 @@ csv_print_cell(csv_cell_t *cell, int with_key) if (with_key) { printf("%s: ", cell->header->field_name); } - switch (cell->header->field_type) - { - case 'i': - printf("%ld", strtol(cell->field_value, NULL, 10)); - break; - - case 'f': - printf("%f", strtod(cell->field_value, NULL)); - break; - - case 's': - printf("%s", cell->field_value); - break; - - default: - printf("%s", cell->field_value); - break; + switch (cell->header->field_type) { + case 'i': + printf("%ld", strtol(cell->field_value, NULL, 10)); + break; + + case 'f': + printf("%f", strtod(cell->field_value, NULL)); + break; + + case 's': + printf("%s", cell->field_value); + break; + + default: + printf("%s", cell->field_value); + break; } - } - - void csv_print_table(csv_table_t *table) { diff --git a/tools/llsm/csvReader.h b/tools/llsm/csvReader.h index e7599ecb2..08a56e4a0 100644 --- a/tools/llsm/csvReader.h +++ b/tools/llsm/csvReader.h @@ -91,7 +91,7 @@ csv_table_t *csv_parse_file(char *file_name, char *field_types); * @param list A PDC_LIST of strings to parse. * @param field_types A string of field types. The field types are 's' for string, 'i' for long integer, 'f' * for float, and 'd' for double. If this is NULL, all fields are assumed to be strings. - * + * * @return A pointer to the csv_table_t struct. */ csv_table_t *csv_parse_list(PDC_LIST *list, char *field_types); diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 727ba02c1..4da885e61 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -781,7 +781,8 @@ _TIFF_load(char *fileName, uint8_t isImageJIm, uint64_t x, uint64_t y, uint64_t } void -parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, image_info_t **image_info) +parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, + image_info_t **image_info) { uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 0, is_imageJ = 0, imageJ_Z = 0; @@ -793,16 +794,17 @@ parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_ dims[1] = flipXY ? x : y; dims[2] = z; - *image_info = (image_info_t *)malloc(sizeof(image_info_t)); - (*image_info)->x = dims[0]; - (*image_info)->y = dims[1]; - (*image_info)->z = dims[2]; - (*image_info)->bits = bits; + *image_info = (image_info_t *)malloc(sizeof(image_info_t)); + (*image_info)->x = dims[0]; + (*image_info)->y = dims[1]; + (*image_info)->z = dims[2]; + (*image_info)->bits = bits; (*image_info)->startSlice = startSlice; (*image_info)->stripeSize = stripeSize; - (*image_info)->is_imageJ = is_imageJ; - (*image_info)->imageJ_Z = imageJ_Z; - (*image_info)->tiff_size = dims[0] * dims[1] * dims[2] * (bits / 8); + (*image_info)->is_imageJ = is_imageJ; + (*image_info)->imageJ_Z = imageJ_Z; + (*image_info)->tiff_size = dims[0] * dims[1] * dims[2] * (bits / 8); - _TIFF_load(fileName, is_imageJ, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, (void **)&((*image_info)->tiff_ptr)); + _TIFF_load(fileName, is_imageJ, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, + (void **)&((*image_info)->tiff_ptr)); } \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h index 60f1cb9f4..081640110 100644 --- a/tools/llsm/parallelReadTiff.h +++ b/tools/llsm/parallelReadTiff.h @@ -28,7 +28,7 @@ typedef struct { size_t tiff_size; } image_info_t; - -void parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, image_info_t **image_info_t); +void parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, + image_info_t **image_info_t); #endif // PARALLELREADTIFF_H \ No newline at end of file diff --git a/tools/llsm/pdc_list.c b/tools/llsm/pdc_list.c index e6ea059be..95c6e59e7 100644 --- a/tools/llsm/pdc_list.c +++ b/tools/llsm/pdc_list.c @@ -1,33 +1,38 @@ #include "pdc_list.h" - -PDC_LIST *pdc_list_new(){ +PDC_LIST * +pdc_list_new() +{ return pdc_list_create(100, 2.0); } -PDC_LIST* pdc_list_create(size_t initial_capacity, double expansion_factor) { +PDC_LIST * +pdc_list_create(size_t initial_capacity, double expansion_factor) +{ // Allocate memory for the list struct. - PDC_LIST* list = (PDC_LIST*) malloc(sizeof(PDC_LIST)); + PDC_LIST *list = (PDC_LIST *)malloc(sizeof(PDC_LIST)); if (list == NULL) { return NULL; } // Allocate memory for the array of items. - list->items = (void**) malloc(initial_capacity * sizeof(void*)); + list->items = (void **)malloc(initial_capacity * sizeof(void *)); if (list->items == NULL) { free(list); return NULL; } // Initialize the list fields. - list->item_count = 0; - list->capacity = initial_capacity; + list->item_count = 0; + list->capacity = initial_capacity; list->expansion_factor = expansion_factor; return list; } -void pdc_list_destroy(PDC_LIST* list) { +void +pdc_list_destroy(PDC_LIST *list) +{ if (list == NULL) { return; } @@ -42,7 +47,9 @@ void pdc_list_destroy(PDC_LIST* list) { free(list); } -void pdc_list_add(PDC_LIST* list, void* item) { +void +pdc_list_add(PDC_LIST *list, void *item) +{ if (list == NULL || item == NULL) { return; } @@ -50,7 +57,7 @@ void pdc_list_add(PDC_LIST* list, void* item) { // Expand the array of items if necessary. if (list->item_count >= list->capacity) { list->capacity *= list->expansion_factor; - list->items = (void**) realloc(list->items, list->capacity * sizeof(void*)); + list->items = (void **)realloc(list->items, list->capacity * sizeof(void *)); if (list->items == NULL) { return; } @@ -60,7 +67,9 @@ void pdc_list_add(PDC_LIST* list, void* item) { list->items[list->item_count++] = item; } -void* pdc_list_get(PDC_LIST* list, size_t index) { +void * +pdc_list_get(PDC_LIST *list, size_t index) +{ if (list == NULL || index >= list->item_count) { return NULL; } @@ -69,7 +78,9 @@ void* pdc_list_get(PDC_LIST* list, size_t index) { return list->items[index]; } -size_t pdc_list_size(PDC_LIST* list) { +size_t +pdc_list_size(PDC_LIST *list) +{ if (list == NULL) { return 0; } @@ -78,7 +89,9 @@ size_t pdc_list_size(PDC_LIST* list) { return list->item_count; } -void pdc_list_set_expansion_factor(PDC_LIST* list, double expansion_factor) { +void +pdc_list_set_expansion_factor(PDC_LIST *list, double expansion_factor) +{ if (list == NULL) { return; } @@ -87,7 +100,9 @@ void pdc_list_set_expansion_factor(PDC_LIST* list, double expansion_factor) { list->expansion_factor = expansion_factor; } -double pdc_list_get_expansion_factor(PDC_LIST* list) { +double +pdc_list_get_expansion_factor(PDC_LIST *list) +{ if (list == NULL) { return 0; } @@ -96,25 +111,29 @@ double pdc_list_get_expansion_factor(PDC_LIST* list) { return list->expansion_factor; } -PDC_LIST_ITERATOR* pdc_list_iterator_new(PDC_LIST* list) { +PDC_LIST_ITERATOR * +pdc_list_iterator_new(PDC_LIST *list) +{ if (list == NULL) { return NULL; } // Allocate memory for the iterator struct. - PDC_LIST_ITERATOR* iterator = (PDC_LIST_ITERATOR*) malloc(sizeof(PDC_LIST_ITERATOR)); + PDC_LIST_ITERATOR *iterator = (PDC_LIST_ITERATOR *)malloc(sizeof(PDC_LIST_ITERATOR)); if (iterator == NULL) { return NULL; } // Initialize the iterator fields. - iterator->list = list; + iterator->list = list; iterator->index = 0; return iterator; } -void pdc_list_iterator_destroy(PDC_LIST_ITERATOR* iterator) { +void +pdc_list_iterator_destroy(PDC_LIST_ITERATOR *iterator) +{ if (iterator == NULL) { return; } @@ -123,7 +142,9 @@ void pdc_list_iterator_destroy(PDC_LIST_ITERATOR* iterator) { free(iterator); } -void* pdc_list_iterator_next(PDC_LIST_ITERATOR* iterator) { +void * +pdc_list_iterator_next(PDC_LIST_ITERATOR *iterator) +{ if (iterator == NULL) { return NULL; } @@ -132,7 +153,9 @@ void* pdc_list_iterator_next(PDC_LIST_ITERATOR* iterator) { return pdc_list_get(iterator->list, iterator->index++); } -int pdc_list_iterator_has_next(PDC_LIST_ITERATOR* iterator) { +int +pdc_list_iterator_has_next(PDC_LIST_ITERATOR *iterator) +{ if (iterator == NULL) { return 0; } diff --git a/tools/llsm/pdc_list.h b/tools/llsm/pdc_list.h index 9847469e5..aa71e6124 100644 --- a/tools/llsm/pdc_list.h +++ b/tools/llsm/pdc_list.h @@ -17,11 +17,10 @@ typedef struct { * A generic iterator for iterating over the items in a PDC_LIST. */ typedef struct { - PDC_LIST *list; // The list being iterated over. - size_t index; // The index of the next item to be returned. + PDC_LIST *list; // The list being iterated over. + size_t index; // The index of the next item to be returned. } PDC_LIST_ITERATOR; - /** * Creates a new PDC_LIST with default initial capacity 100 and default expansion factor 2.0. * @return A pointer to the new PDC_LIST. @@ -32,7 +31,7 @@ PDC_LIST *pdc_list_new(); * Creates a new PDC_LIST with the given initial capacity and expansion factor. * @param initial_capacity The initial capacity of the list. * @param expansion_factor The factor by which the capacity is expanded when the list is full. - * + * * @return A pointer to the new PDC_LIST. */ PDC_LIST *pdc_list_create(size_t initial_capacity, double expansion_factor); @@ -47,7 +46,7 @@ void pdc_list_destroy(PDC_LIST *list); * Adds the given item to the end of the given PDC_LIST. * @param list The PDC_LIST to add the item to. * @param item The item to add to the PDC_LIST. - * + * */ void pdc_list_add(PDC_LIST *list, void *item); @@ -55,7 +54,7 @@ void pdc_list_add(PDC_LIST *list, void *item); * Gets the item at the given index in the given PDC_LIST. * @param list The PDC_LIST to get the item from. * @param index The index of the item to get. - * + * * @return A pointer to the item at the given index. */ void *pdc_list_get(PDC_LIST *list, size_t index); @@ -63,7 +62,7 @@ void *pdc_list_get(PDC_LIST *list, size_t index); /** * Sets the item at the given index in the given PDC_LIST. * @param list The PDC_LIST to set the item in. - * + * * @return The number of items in the list. */ size_t pdc_list_size(PDC_LIST *list); diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 6ac2cfa2c..8ac8fc9fc 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -246,7 +246,6 @@ main(int argc, char *argv[]) printf("Filename: %s\n", file_name ? file_name : "(none)"); printf("Directory: %s\n", directory_path ? directory_path : "(none)"); - // // create a pdc // pdc_id_g = PDCinit("pdc"); @@ -260,7 +259,6 @@ main(int argc, char *argv[]) // if (cont <= 0) // printf("Fail to create container @ line %d!\n", __LINE__); - // Rank 0 reads the filename list and distribute data to other ranks if (rank == 0) { read_txt(file_name, list); From 0e13170103966decdca5cf0a29196446157ad5d6 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:32:14 -0500 Subject: [PATCH 138/216] enable MPI --- tools/llsm_importer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 8ac8fc9fc..cdd5ddd56 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -9,8 +9,8 @@ #endif #ifdef ENABLE_MPI -// #include "mpi.h" -#undef ENABLE_MPI +#include "mpi.h" +// #undef ENABLE_MPI #endif #include "pdc.h" From bdfd297c23014747a57abc2e962425d853ffafcb Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:37:04 -0500 Subject: [PATCH 139/216] enable MPI --- tools/llsm/csvReader.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/llsm/csvReader.c b/tools/llsm/csvReader.c index 39c4571aa..bb52748e3 100644 --- a/tools/llsm/csvReader.c +++ b/tools/llsm/csvReader.c @@ -296,9 +296,9 @@ csv_print_row(csv_row_t *row, int with_key) csv_print_cell(current_cell, with_key); if (current_cell->next != NULL) { printf(", "); - if (with_key) { - printf("\n"); - } + } + if (with_key) { + printf("\n"); } current_cell = current_cell->next; } From 62b0efbf9674b6415e7065522442161e3afa934d Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:42:05 -0500 Subject: [PATCH 140/216] enlarge BCase size --- tools/llsm_importer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index cdd5ddd56..b75ec349a 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -270,7 +270,7 @@ main(int argc, char *argv[]) PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); while (pdc_list_iterator_has_next(iter)) { char *csv_line = (char *)pdc_list_iterator_next(iter); - MPI_Bcast(csv_line, 256, MPI_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(csv_line, 1024, MPI_CHAR, 0, MPI_COMM_WORLD); } #endif } @@ -282,8 +282,8 @@ main(int argc, char *argv[]) // receive the file names int i; for (i = 0; i < num_lines; i++) { - csv_line = (char *)malloc(256 * sizeof(char)); - MPI_Bcast(csv_line, 256, MPI_CHAR, 0, MPI_COMM_WORLD); + csv_line = (char *)malloc(1024 * sizeof(char)); + MPI_Bcast(csv_line, 1024, MPI_CHAR, 0, MPI_COMM_WORLD); pdc_list_add(list, csv_line); } #endif From 399289cad8236741c78f3ceddc1725729c7705f3 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:47:28 -0500 Subject: [PATCH 141/216] enlarge BCase size --- tools/llsm_importer.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index b75ec349a..00e29e413 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -202,10 +202,12 @@ on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) } void -read_txt(char *txtFileName, PDC_LIST *list) +read_txt(char *txtFileName, PDC_LIST *list, int *max_row_length) { FILE *file = fopen(txtFileName, "r"); + int row_length = 0; + if (file == NULL) { printf("Error: could not open file %s\n", txtFileName); return; @@ -214,7 +216,15 @@ read_txt(char *txtFileName, PDC_LIST *list) // Read the lines of the file while (fgets(buffer, sizeof(buffer), file)) { pdc_list_add(list, strdup(buffer)); + if (row_length < strlen(buffer)) { + row_length = strlen(buffer); + } } + + fclose(file); + + // Find the maximum row length + *max_row_length = row_length + 5; } int @@ -234,6 +244,7 @@ main(int argc, char *argv[]) csv_header_t * csv_header = NULL; csv_row_t * csv_row = NULL; llsm_importer_args_t *llsm_args = NULL; + int bcast_count = 512; char csv_field_types[] = {'s', 's', 'f', 'f', 'f', 'f', 'f', 'f'}; // parse console argument int parse_code = parse_console_args(argc, argv, &file_name); @@ -261,7 +272,7 @@ main(int argc, char *argv[]) // Rank 0 reads the filename list and distribute data to other ranks if (rank == 0) { - read_txt(file_name, list); + read_txt(file_name, list, &bcast_count); #ifdef ENABLE_MPI // broadcast the number of lines int num_lines = pdc_list_size(list); @@ -270,7 +281,7 @@ main(int argc, char *argv[]) PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); while (pdc_list_iterator_has_next(iter)) { char *csv_line = (char *)pdc_list_iterator_next(iter); - MPI_Bcast(csv_line, 1024, MPI_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(csv_line, bcast_count, MPI_CHAR, 0, MPI_COMM_WORLD); } #endif } @@ -282,8 +293,8 @@ main(int argc, char *argv[]) // receive the file names int i; for (i = 0; i < num_lines; i++) { - csv_line = (char *)malloc(1024 * sizeof(char)); - MPI_Bcast(csv_line, 1024, MPI_CHAR, 0, MPI_COMM_WORLD); + csv_line = (char *)malloc(bcast_count * sizeof(char)); + MPI_Bcast(csv_line, bcast_count, MPI_CHAR, 0, MPI_COMM_WORLD); pdc_list_add(list, csv_line); } #endif From ab24b6bb92f1b346380d29be63ed3aa883e91f12 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:51:02 -0500 Subject: [PATCH 142/216] enlarge BCase size --- tools/llsm_importer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 00e29e413..795e31490 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -281,7 +281,7 @@ main(int argc, char *argv[]) PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); while (pdc_list_iterator_has_next(iter)) { char *csv_line = (char *)pdc_list_iterator_next(iter); - MPI_Bcast(csv_line, bcast_count, MPI_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(csv_line, 512, MPI_CHAR, 0, MPI_COMM_WORLD); } #endif } @@ -293,8 +293,8 @@ main(int argc, char *argv[]) // receive the file names int i; for (i = 0; i < num_lines; i++) { - csv_line = (char *)malloc(bcast_count * sizeof(char)); - MPI_Bcast(csv_line, bcast_count, MPI_CHAR, 0, MPI_COMM_WORLD); + csv_line = (char *)malloc(512 * sizeof(char)); + MPI_Bcast(csv_line, 512, MPI_CHAR, 0, MPI_COMM_WORLD); pdc_list_add(list, csv_line); } #endif From bd97115d52c299c9821e01ccb1c8bbf42f282ea0 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 18:55:27 -0500 Subject: [PATCH 143/216] resolve bcast count --- tools/llsm_importer.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 795e31490..a0671b51d 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -273,15 +273,20 @@ main(int argc, char *argv[]) // Rank 0 reads the filename list and distribute data to other ranks if (rank == 0) { read_txt(file_name, list, &bcast_count); + // print bcast_count + printf("bcast_count: %d", bcast_count); + #ifdef ENABLE_MPI // broadcast the number of lines int num_lines = pdc_list_size(list); MPI_Bcast(&num_lines, 1, MPI_INT, 0, MPI_COMM_WORLD); + // broadcast the bcast_count + MPI_Bcast(&bcast_count, 1, MPI_INT, 0, MPI_COMM_WORLD); // broadcast the file names PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); while (pdc_list_iterator_has_next(iter)) { char *csv_line = (char *)pdc_list_iterator_next(iter); - MPI_Bcast(csv_line, 512, MPI_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(csv_line, bcast_count, MPI_CHAR, 0, MPI_COMM_WORLD); } #endif } @@ -290,11 +295,13 @@ main(int argc, char *argv[]) // other ranks receive the number of files int num_lines; MPI_Bcast(&num_lines, 1, MPI_INT, 0, MPI_COMM_WORLD); + // receive the bcast_count + MPI_Bcast(&bcast_count, 1, MPI_INT, 0, MPI_COMM_WORLD); // receive the file names int i; for (i = 0; i < num_lines; i++) { - csv_line = (char *)malloc(512 * sizeof(char)); - MPI_Bcast(csv_line, 512, MPI_CHAR, 0, MPI_COMM_WORLD); + csv_line = (char *)malloc(bcast_count * sizeof(char)); + MPI_Bcast(csv_line, bcast_count, MPI_CHAR, 0, MPI_COMM_WORLD); pdc_list_add(list, csv_line); } #endif From ed20ee04e35f90ec23fdbb0f707497c8c9bb01ec Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 19:05:02 -0500 Subject: [PATCH 144/216] llsm data path in script --- scripts/llsm_importer/template.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/llsm_importer/template.sh b/scripts/llsm_importer/template.sh index d9305c83e..d736b4076 100755 --- a/scripts/llsm_importer/template.sh +++ b/scripts/llsm_importer/template.sh @@ -38,7 +38,9 @@ CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* chmod +x $TOOLPATH/llsm_importer -IMGLIST_PATH=/global/cfs/cdirs/m2621/wzhang5/data/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1/ImageList_from_encoder.csv +LLSM_DATA_PATH=/pscratch/sd/w/wzhang5/data/llsm/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1 +# LLSM_DATA_PATH=/global/cfs/cdirs/m2621/wzhang5/data/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1 +IMGLIST_PATH=${LLSM_DATA_PATH}/ImageList_from_encoder.csv date From 29bb5259e311727f4fec03938f1dac619aee60f0 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 19:51:33 -0500 Subject: [PATCH 145/216] llsm data path in script --- tools/llsm/csvReader.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/llsm/csvReader.c b/tools/llsm/csvReader.c index bb52748e3..5823d349a 100644 --- a/tools/llsm/csvReader.c +++ b/tools/llsm/csvReader.c @@ -297,9 +297,9 @@ csv_print_row(csv_row_t *row, int with_key) if (current_cell->next != NULL) { printf(", "); } - if (with_key) { - printf("\n"); - } + // if (with_key) { + // printf("\n"); + // } current_cell = current_cell->next; } printf("\n"); From 8876a4353dcf4f16ee9d17d3b7eaa5eb300a32db Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 20:07:36 -0500 Subject: [PATCH 146/216] update csv reader --- tools/llsm/csvReader.c | 45 ++++++++++++++++++++++++++++++++++-------- tools/llsm/csvReader.h | 25 +++++++++++++++++++++++ tools/llsm_importer.c | 2 +- 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/tools/llsm/csvReader.c b/tools/llsm/csvReader.c index 5823d349a..c4f4fbf81 100644 --- a/tools/llsm/csvReader.c +++ b/tools/llsm/csvReader.c @@ -1,5 +1,34 @@ #include "csvReader.h" +char csv_delimiter = ','; +char csv_quote = '\"'; +char csv_escape = '\\'; +char csv_newline = '\n'; + +void +csv_set_delimiter(char delimiter) +{ + csv_delimiter = delimiter; +} + +void +csv_set_quote(char quote) +{ + csv_quote = quote; +} + +void +csv_set_escape(char escape) +{ + csv_escape = escape; +} + +void +csv_set_newline(char newline) +{ + csv_newline = newline; +} + csv_header_t * csv_parse_header(char *line, char *field_types) { @@ -12,18 +41,18 @@ csv_parse_header(char *line, char *field_types) int value_start = 0; int i = 0; - for (int i = 0; line[i] != '\0'; ++i) { - if (line[i] == '\"') { + for (int i = 0; line[i] != csv_newline; ++i) { + if (line[i] == csv_quote) { in_quotes = !in_quotes; } - else if (!in_quotes && (line[i] == ',' || line[i + 1] == '\0')) { + else if (!in_quotes && (line[i] == csv_delimiter || line[i + 1] == csv_newline)) { // Allocate memory for the header struct csv_header_t *header = (csv_header_t *)malloc(sizeof(csv_header_t)); if (header == NULL) { return NULL; } // Remove quotes and spaces from the field name - header->field_name = strndup(line + value_start, i - value_start + (line[i + 1] == '\0')); + header->field_name = strndup(line + value_start, i - value_start + (line[i + 1] == csv_newline)); // Set the field index header->field_index = field_index; @@ -70,11 +99,11 @@ csv_parse_row(char *line, csv_header_t *header) int value_start = 0; int i = 0; - for (int i = 0; line[i] != '\0'; ++i) { - if (line[i] == '\"') { + for (int i = 0; line[i] != csv_newline; ++i) { + if (line[i] == csv_quote) { in_quotes = !in_quotes; } - else if (!in_quotes && (line[i] == ',' || line[i + 1] == '\0')) { + else if (!in_quotes && (line[i] == csv_delimiter || line[i + 1] == csv_newline)) { // Allocate memory for the cell struct csv_cell_t *cell = (csv_cell_t *)malloc(sizeof(csv_cell_t)); if (cell == NULL) { @@ -85,7 +114,7 @@ csv_parse_row(char *line, csv_header_t *header) cell->header = current_header; // Set the field value - cell->field_value = strndup(line + value_start, i - value_start + (line[i + 1] == '\0')); + cell->field_value = strndup(line + value_start, i - value_start + (line[i + 1] == csv_newline)); // Set the next pointer to NULL cell->next = NULL; diff --git a/tools/llsm/csvReader.h b/tools/llsm/csvReader.h index 08a56e4a0..898f60fdb 100644 --- a/tools/llsm/csvReader.h +++ b/tools/llsm/csvReader.h @@ -30,6 +30,31 @@ typedef struct csv_table_t { csv_row_t * first_row; } csv_table_t; +/** + * @brief This function sets the delimiter for the CSV file. The default is a comma. + * @param delimiter The delimiter to use. + */ +void csv_set_delimiter(char delimiter); + +/** + * @brief This function sets the quote character for the CSV file. The default is a double quote. + * @param quote The quote character to use. + */ +void csv_set_quote(char quote); + +/** + * @brief This function sets the escape character for the CSV file. The default is a backslash. + * @param escape The escape character to use. + */ +void csv_set_escape(char escape); + +/** + * @brief This function sets the newline character for the CSV file. The default is a newline. + * @param newline The newline character to use. + */ +void csv_set_newline(char newline); + + /** * @brief This function parses a CSV header line and returns a linked list of csv_header_t structs. The header * string may contain quotes and spaces diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index a0671b51d..f9a78a7da 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -274,7 +274,7 @@ main(int argc, char *argv[]) if (rank == 0) { read_txt(file_name, list, &bcast_count); // print bcast_count - printf("bcast_count: %d", bcast_count); + printf("bcast_count: %d \n", bcast_count); #ifdef ENABLE_MPI // broadcast the number of lines From 39f396b596d2f5416df144244b8999dbb4fe95aa Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 20:08:37 -0500 Subject: [PATCH 147/216] update csv reader --- tools/llsm/csvReader.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/llsm/csvReader.c b/tools/llsm/csvReader.c index c4f4fbf81..0bc480edf 100644 --- a/tools/llsm/csvReader.c +++ b/tools/llsm/csvReader.c @@ -325,10 +325,10 @@ csv_print_row(csv_row_t *row, int with_key) csv_print_cell(current_cell, with_key); if (current_cell->next != NULL) { printf(", "); + if (with_key) { + printf("\n"); + } } - // if (with_key) { - // printf("\n"); - // } current_cell = current_cell->next; } printf("\n"); From e587abf11e2bc12267ff1ad0d0540093fde24fcc Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 20:11:10 -0500 Subject: [PATCH 148/216] update csv reader --- tools/llsm/csvReader.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/llsm/csvReader.c b/tools/llsm/csvReader.c index 0bc480edf..c6623a287 100644 --- a/tools/llsm/csvReader.c +++ b/tools/llsm/csvReader.c @@ -325,9 +325,9 @@ csv_print_row(csv_row_t *row, int with_key) csv_print_cell(current_cell, with_key); if (current_cell->next != NULL) { printf(", "); - if (with_key) { - printf("\n"); - } + } + if (with_key) { + printf("\n"); } current_cell = current_cell->next; } From 3aa45056a4a8ec3122ffdf969956a2a133ccf36a Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 23:51:28 -0500 Subject: [PATCH 149/216] update pdc --- tools/llsm_importer.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index f9a78a7da..ef13c5dde 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -29,7 +29,6 @@ typedef struct llsm_importer_args_t { int rank = 0, size = 1; -pdcid_t pdc, cont_prop, cont, obj_prop; pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; int @@ -60,7 +59,7 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation - pdcid_t cur_obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); + obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); psize_t ndims = 3; // FIXME: we should support uint64_t. @@ -82,7 +81,7 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) // create object // FIXME: There are many attributes currently in one file name, // and we should do some research to see what would be a good object name for each image. - pdcid_t cur_obj_g = PDCobj_create(cont_id_g, fileName_cell->field_value, cur_obj_prop_g); + pdcid_t cur_obj_g = PDCobj_create(cont_id_g, fileName_cell->field_value, obj_prop_g); // write data to object pdcid_t local_region = PDCregion_create(ndims, offsets, num_bytes); @@ -143,7 +142,7 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) PDCregion_close(local_region); PDCregion_close(remote_region); PDCregion_transfer_close(transfer_request); - PDCprop_close(cur_obj_prop_g); + PDCprop_close(obj_prop_g); } void @@ -257,18 +256,18 @@ main(int argc, char *argv[]) printf("Filename: %s\n", file_name ? file_name : "(none)"); printf("Directory: %s\n", directory_path ? directory_path : "(none)"); - // // create a pdc - // pdc_id_g = PDCinit("pdc"); + // create a pdc + pdc_id_g = PDCinit("pdc"); - // // create a container property - // cont_prop_g = PDCprop_create(PDC_CONT_CREATE, pdc); - // if (cont_prop <= 0) - // printf("Fail to create container property @ line %d!\n", __LINE__); + // create a container property + cont_prop_g = PDCprop_create(PDC_CONT_CREATE, pdc_id_g); + if (cont_prop_g <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); - // // create a container - // cont = PDCcont_create("c1", cont_prop); - // if (cont <= 0) - // printf("Fail to create container @ line %d!\n", __LINE__); + // create a container + cont_id_g = PDCcont_create("c1", cont_prop_g); + if (cont_id_g <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); // Rank 0 reads the filename list and distribute data to other ranks if (rank == 0) { @@ -328,6 +327,13 @@ main(int argc, char *argv[]) csv_free_table(csv_table); + // close the container + PDCcont_close(cont_id_g); + // close the container property + PDCprop_close(cont_prop_g); + // close the pdc + PDCclose(pdc_id_g); + #ifdef ENABLE_MPI MPI_Finalize(); #endif From 2154141c0e0dc977aeff78b68856a60a751e968f Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 23:51:44 -0500 Subject: [PATCH 150/216] update pdc --- tools/llsm_importer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index ef13c5dde..894220403 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -192,7 +192,7 @@ on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) printf("\n"); // import the image to PDC - // import_to_pdc(image_info, fileName_cell); + import_to_pdc(image_info, fileName_cell); // free the image info free(image_info->tiff_ptr); From 8f559748a7e9cfd2585c369e8c3893a2db77d207 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 23 Apr 2023 23:54:54 -0500 Subject: [PATCH 151/216] update pdc --- tools/llsm_importer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 894220403..fc6a27148 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -9,8 +9,8 @@ #endif #ifdef ENABLE_MPI -#include "mpi.h" -// #undef ENABLE_MPI +// #include "mpi.h" +#undef ENABLE_MPI #endif #include "pdc.h" From d8b2e572d0bf6703f8826776a8ad30dd19e2c38e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 00:29:12 -0500 Subject: [PATCH 152/216] update pdc --- tools/llsm_importer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index fc6a27148..b1b240a14 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -61,9 +61,9 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); - psize_t ndims = 3; + psize_t ndims = 1; // FIXME: we should support uint64_t. - uint64_t dims[3] = {image_info->x, image_info->y, image_info->z}; + uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; // FIXME: we should change the ndims parameter to psize_t type. PDCprop_set_obj_dims(obj_prop_g, (PDC_int_t)ndims, dims); PDCprop_set_obj_type(obj_prop_g, PDC_FLOAT); From 3009e0bcd5a4c3ed28699949ec50fdf1904a6557 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 00:32:34 -0500 Subject: [PATCH 153/216] update pdc --- tools/llsm_importer.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index b1b240a14..8e30e4718 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -84,12 +84,12 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) pdcid_t cur_obj_g = PDCobj_create(cont_id_g, fileName_cell->field_value, obj_prop_g); // write data to object - pdcid_t local_region = PDCregion_create(ndims, offsets, num_bytes); - pdcid_t remote_region = PDCregion_create(ndims, offsets, num_bytes); - pdcid_t transfer_request = - PDCregion_transfer_create(image_info->tiff_ptr, PDC_WRITE, cur_obj_g, local_region, remote_region); - PDCregion_transfer_start(transfer_request); - PDCregion_transfer_wait(transfer_request); + // pdcid_t local_region = PDCregion_create(ndims, offsets, num_bytes); + // pdcid_t remote_region = PDCregion_create(ndims, offsets, num_bytes); + // pdcid_t transfer_request = + // PDCregion_transfer_create(image_info->tiff_ptr, PDC_WRITE, cur_obj_g, local_region, remote_region); + // PDCregion_transfer_start(transfer_request); + // PDCregion_transfer_wait(transfer_request); // add metadata tags based on the csv row csv_cell_t *cell = fileName_cell; @@ -139,9 +139,9 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) // free memory free(offsets); free(num_bytes); - PDCregion_close(local_region); - PDCregion_close(remote_region); - PDCregion_transfer_close(transfer_request); + // PDCregion_close(local_region); + // PDCregion_close(remote_region); + // PDCregion_transfer_close(transfer_request); PDCprop_close(obj_prop_g); } From b90191f38431b5f3af02a4850e170035f64c6fa9 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 15:24:25 -0500 Subject: [PATCH 154/216] update pdc --- tools/llsm_importer.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 8e30e4718..85bfa3fef 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -61,9 +61,16 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); - psize_t ndims = 1; + psize_t ndims = 3; + uint64_t offsets[3] = {0, 0, 0}; // FIXME: we should support uint64_t. - uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; + uint64_t dims[3] = {image_info->x , image_info->y , image_info->z}; + + // psize_t ndims = 1; + // uint64_t offsets[1] = {0}; + // // FIXME: we should support uint64_t. + // uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; + // FIXME: we should change the ndims parameter to psize_t type. PDCprop_set_obj_dims(obj_prop_g, (PDC_int_t)ndims, dims); PDCprop_set_obj_type(obj_prop_g, PDC_FLOAT); @@ -71,12 +78,12 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) PDCprop_set_obj_user_id(obj_prop_g, getuid()); PDCprop_set_obj_app_name(obj_prop_g, "LLSM"); - uint64_t *offsets = (uint64_t *)malloc(sizeof(uint64_t) * ndims); - uint64_t *num_bytes = (uint64_t *)malloc(sizeof(uint64_t) * ndims); - for (int i = 0; i < ndims; i++) { - offsets[i] = 0; - num_bytes[i] = dims[i] * image_info->bits / 8; - } + // uint64_t *offsets = (uint64_t *)malloc(sizeof(uint64_t) * ndims); + // uint64_t *num_bytes = (uint64_t *)malloc(sizeof(uint64_t) * ndims); + // for (int i = 0; i < ndims; i++) { + // offsets[i] = 0; + // num_bytes[i] = dims[i] * image_info->bits / 8; + // } // create object // FIXME: There are many attributes currently in one file name, @@ -84,12 +91,12 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) pdcid_t cur_obj_g = PDCobj_create(cont_id_g, fileName_cell->field_value, obj_prop_g); // write data to object - // pdcid_t local_region = PDCregion_create(ndims, offsets, num_bytes); - // pdcid_t remote_region = PDCregion_create(ndims, offsets, num_bytes); - // pdcid_t transfer_request = - // PDCregion_transfer_create(image_info->tiff_ptr, PDC_WRITE, cur_obj_g, local_region, remote_region); - // PDCregion_transfer_start(transfer_request); - // PDCregion_transfer_wait(transfer_request); + pdcid_t local_region = PDCregion_create(ndims, offsets, dims); + pdcid_t remote_region = PDCregion_create(ndims, offsets, dims); + pdcid_t transfer_request = + PDCregion_transfer_create(image_info->tiff_ptr, PDC_WRITE, cur_obj_g, local_region, remote_region); + PDCregion_transfer_start(transfer_request); + PDCregion_transfer_wait(transfer_request); // add metadata tags based on the csv row csv_cell_t *cell = fileName_cell; From 08f9fc226fb0c7236a74e34dcfc2281fa0fe2fa7 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 15:26:33 -0500 Subject: [PATCH 155/216] update pdc --- tools/llsm_importer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 85bfa3fef..28f57777d 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -144,8 +144,8 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) duration / 1e9); // free memory - free(offsets); - free(num_bytes); + // free(offsets); + // free(num_bytes); // PDCregion_close(local_region); // PDCregion_close(remote_region); // PDCregion_transfer_close(transfer_request); From a3dc4972cecdc4259def3b6e0d60d83756364ea3 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 15:31:58 -0500 Subject: [PATCH 156/216] update pdc --- tools/llsm_importer.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 28f57777d..af08d50d2 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -61,15 +61,15 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); - psize_t ndims = 3; - uint64_t offsets[3] = {0, 0, 0}; - // FIXME: we should support uint64_t. - uint64_t dims[3] = {image_info->x , image_info->y , image_info->z}; - - // psize_t ndims = 1; - // uint64_t offsets[1] = {0}; + // psize_t ndims = 3; + // uint64_t offsets[3] = {0, 0, 0}; // // FIXME: we should support uint64_t. - // uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; + // uint64_t dims[3] = {image_info->x , image_info->y , image_info->z}; + + psize_t ndims = 1; + uint64_t offsets[1] = {0}; + // FIXME: we should support uint64_t. + uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; // FIXME: we should change the ndims parameter to psize_t type. PDCprop_set_obj_dims(obj_prop_g, (PDC_int_t)ndims, dims); From 2e1c8bc77ff02cf14a359b26d6ba0ff2133a310f Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 15:43:31 -0500 Subject: [PATCH 157/216] update pdc --- tools/llsm_importer.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index af08d50d2..dcb0cba21 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -73,7 +73,25 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) // FIXME: we should change the ndims parameter to psize_t type. PDCprop_set_obj_dims(obj_prop_g, (PDC_int_t)ndims, dims); - PDCprop_set_obj_type(obj_prop_g, PDC_FLOAT); + pdc_var_type_t pdc_type = PDC_UNKNOWN; + switch (image_info->bits) { + case 8: + pdc_type = PDC_INT8; + break; + case 16: + pdc_type = PDC_INT16; + break; + case 32: + pdc_type = PDC_FLOAT; + break; + case 64: + pdc_type = PDC_DOUBLE; + break; + default: + printf("Error: unsupported data type.\n"); + exit(-1); + } + PDCprop_set_obj_type(obj_prop_g, pdc_type); PDCprop_set_obj_time_step(obj_prop_g, 0); PDCprop_set_obj_user_id(obj_prop_g, getuid()); PDCprop_set_obj_app_name(obj_prop_g, "LLSM"); From 3584a20bfe35ddf882adb2b74d325ffe78425821 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 15:51:51 -0500 Subject: [PATCH 158/216] update pdc --- src/api/include/pdc_public.h | 4 +++- src/server/pdc_client_server_common.c | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/api/include/pdc_public.h b/src/api/include/pdc_public.h index dc286e5e9..8c47976f0 100644 --- a/src/api/include/pdc_public.h +++ b/src/api/include/pdc_public.h @@ -55,7 +55,9 @@ typedef enum { PDC_UINT64 = 9, /* 64-bit unsigned integer types */ PDC_INT16 = 10, PDC_INT8 = 11, - NCLASSES = 12 /* this must be last */ + PDC_UINT8 = 12, + PDC_UINT16 = 13, + NCLASSES = 14 /* this must be last */ } pdc_var_type_t; typedef enum { PDC_PERSIST, PDC_TRANSIENT } pdc_lifetime_t; diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index 289c20527..14417485b 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -265,7 +265,14 @@ PDC_get_var_type_size(pdc_var_type_t dtype) break; case PDC_INT8: ret_value = sizeof(int8_t); - + goto done; + break; + case PDC_UINT8: + ret_value = sizeof(uint8_t); + goto done; + break; + case PDC_UINT16: + ret_value = sizeof(uint16_t); goto done; break; case PDC_INT64: From 5b5e90e54d26f8a438f96e4e62d41ba21e78a44b Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 15:54:55 -0500 Subject: [PATCH 159/216] update pdc --- tools/llsm_importer.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index dcb0cba21..a412eda52 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -61,25 +61,25 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); - // psize_t ndims = 3; - // uint64_t offsets[3] = {0, 0, 0}; - // // FIXME: we should support uint64_t. - // uint64_t dims[3] = {image_info->x , image_info->y , image_info->z}; - - psize_t ndims = 1; - uint64_t offsets[1] = {0}; + psize_t ndims = 3; + uint64_t offsets[3] = {0, 0, 0}; // FIXME: we should support uint64_t. - uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; + uint64_t dims[3] = {image_info->x , image_info->y , image_info->z}; + + // psize_t ndims = 1; + // uint64_t offsets[1] = {0}; + // // FIXME: we should support uint64_t. + // uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; // FIXME: we should change the ndims parameter to psize_t type. PDCprop_set_obj_dims(obj_prop_g, (PDC_int_t)ndims, dims); pdc_var_type_t pdc_type = PDC_UNKNOWN; switch (image_info->bits) { case 8: - pdc_type = PDC_INT8; + pdc_type = PDC_UINT8; break; case 16: - pdc_type = PDC_INT16; + pdc_type = PDC_UINT16; break; case 32: pdc_type = PDC_FLOAT; From 53c07d81de7275d1bb4facf5a25e594bd1048ee0 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 15:59:37 -0500 Subject: [PATCH 160/216] enlarge max write --- src/server/pdc_server_region/pdc_server_data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/pdc_server_region/pdc_server_data.c b/src/server/pdc_server_region/pdc_server_data.c index c8f80278a..11d91dc22 100644 --- a/src/server/pdc_server_region/pdc_server_data.c +++ b/src/server/pdc_server_region/pdc_server_data.c @@ -4686,7 +4686,7 @@ static perr_t PDC_Server_posix_write(int fd, void *buf, uint64_t write_size) { // Write 1GB at a time - uint64_t write_bytes = 0, max_write_size = 1073741824; + uint64_t write_bytes = 0, max_write_size = 1073741824 * 2; perr_t ret_value = SUCCEED; ssize_t ret; From c7551eaa5e076ce5f2c488a75ec0e5fe3da5674e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 16:17:58 -0500 Subject: [PATCH 161/216] update pdc --- tools/llsm_importer.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index a412eda52..1b86081d2 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -61,16 +61,16 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); - psize_t ndims = 3; + psize_t ndims = 3; uint64_t offsets[3] = {0, 0, 0}; // FIXME: we should support uint64_t. - uint64_t dims[3] = {image_info->x , image_info->y , image_info->z}; + uint64_t dims[3] = {image_info->x, image_info->y, image_info->z}; // psize_t ndims = 1; // uint64_t offsets[1] = {0}; // // FIXME: we should support uint64_t. // uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; - + // FIXME: we should change the ndims parameter to psize_t type. PDCprop_set_obj_dims(obj_prop_g, (PDC_int_t)ndims, dims); pdc_var_type_t pdc_type = PDC_UNKNOWN; @@ -116,6 +116,13 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) PDCregion_transfer_start(transfer_request); PDCregion_transfer_wait(transfer_request); + clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation + duration = (end.tv_sec - start.tv_sec) * 1e9 + + (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds + + printf("[Rank %4d] Region Transfer for object %s [%d Bytes] Done! Time taken: %.4f seconds\n", rank, + fileName_cell->field_value, image_info->tiff_size , duration / 1e9); + // add metadata tags based on the csv row csv_cell_t *cell = fileName_cell; while (cell != NULL) { @@ -158,7 +165,7 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) duration = (end.tv_sec - start.tv_sec) * 1e9 + (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds - printf("[Rank %4d]create object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, + printf("[Rank %4d] Create object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, duration / 1e9); // free memory @@ -204,7 +211,7 @@ on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) duration = (end.tv_sec - start.tv_sec) * 1e9 + (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds - printf("[Rand %4d]Read %s Done! Time taken: %.4f seconds\n", rank, filepath, duration / 1e9); + printf("[Rand %4d] Read %s Done! Time taken: %.4f seconds\n", rank, filepath, duration / 1e9); if (image_info == NULL || image_info->tiff_ptr == NULL) { return; From 02d401fc72874882500e9a312a251b576a586ea6 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 16:23:59 -0500 Subject: [PATCH 162/216] update pdc --- tools/llsm_importer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 1b86081d2..07196d018 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -9,8 +9,8 @@ #endif #ifdef ENABLE_MPI -// #include "mpi.h" -#undef ENABLE_MPI +#include "mpi.h" +// #undef ENABLE_MPI #endif #include "pdc.h" From cdc30203bfa1c75950944045ce273b2e454ea83f Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 16:34:25 -0500 Subject: [PATCH 163/216] update pdc --- src/server/pdc_server_region/pdc_server_data.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/server/pdc_server_region/pdc_server_data.c b/src/server/pdc_server_region/pdc_server_data.c index 11d91dc22..7d4057b1e 100644 --- a/src/server/pdc_server_region/pdc_server_data.c +++ b/src/server/pdc_server_region/pdc_server_data.c @@ -4695,7 +4695,8 @@ PDC_Server_posix_write(int fd, void *buf, uint64_t write_size) while (write_size > max_write_size) { ret = write(fd, buf, max_write_size); if (ret < 0 || ret != (ssize_t)max_write_size) { - printf("==PDC_SERVER[%d]: write %d failed\n", pdc_server_rank_g, fd); + printf("==PDC_SERVER[%d]: in-loop: write %d failed, ret = %d, max_write_size = %llu\n", + pdc_server_rank_g, fd, ret, max_write_size); ret_value = FAIL; goto done; } @@ -4705,7 +4706,8 @@ PDC_Server_posix_write(int fd, void *buf, uint64_t write_size) } ret = write(fd, buf, write_size); if (ret < 0 || ret != (ssize_t)write_size) { - printf("==PDC_SERVER[%d]: write %d failed\n", pdc_server_rank_g, fd); + printf("==PDC_SERVER[%d]: out-loop: write %d failed, ret = %d, max_write_size = %llu\n", + pdc_server_rank_g, fd, ret, max_write_size); ret_value = FAIL; goto done; } From 6d7d7a3549cd2ec3df3145a661664b3d326e110b Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Apr 2023 22:39:47 -0500 Subject: [PATCH 164/216] update pdc --- src/server/pdc_server_region/pdc_server_data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/pdc_server_region/pdc_server_data.c b/src/server/pdc_server_region/pdc_server_data.c index 7d4057b1e..372d08ac7 100644 --- a/src/server/pdc_server_region/pdc_server_data.c +++ b/src/server/pdc_server_region/pdc_server_data.c @@ -4686,7 +4686,7 @@ static perr_t PDC_Server_posix_write(int fd, void *buf, uint64_t write_size) { // Write 1GB at a time - uint64_t write_bytes = 0, max_write_size = 1073741824 * 2; + uint64_t write_bytes = 0, max_write_size = 1073741824; perr_t ret_value = SUCCEED; ssize_t ret; From 011b9f080baba9930dfdf4909f38b66b179079f3 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 25 Apr 2023 22:18:32 -0500 Subject: [PATCH 165/216] update pdc_import.c --- tools/CMakeLists.txt | 25 ++++++++++++------------- tools/pdc_import.c | 29 ++++++++++++++++------------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1c6940ca9..815a081ec 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -87,20 +87,19 @@ add_definitions(-DENABLE_MPI=1) add_library(cjson cjson/cJSON.c) -# set(PROGRAMS -# pdc_ls -# pdc_import -# pdc_export -# ) +set(PROGRAMS + pdc_ls + pdc_import + pdc_export + ) -# foreach(program ${PROGRAMS}) -# add_executable(${program} ${program}.c) -# target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) -# target_link_libraries(${program} pdc) -# target_link_libraries(${program} cjson) -# target_link_libraries(${program} llsm_tiff) -# target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) -# endforeach(program) +foreach(program ${PROGRAMS}) + add_executable(${program} ${program}.c) + target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) + target_link_libraries(${program} pdc) + target_link_libraries(${program} cjson) + target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) +endforeach(program) # Find LibTIFF diff --git a/tools/pdc_import.c b/tools/pdc_import.c index 9388ac495..4a0af982c 100644 --- a/tools/pdc_import.c +++ b/tools/pdc_import.c @@ -10,8 +10,8 @@ #include "hdf5.h" #include "pdc.h" -#include "pdc_client_server_common.h" -#include "pdc_client_connect.h" +// #include "pdc_client_server_common.h" +// #include "pdc_client_connect.h" #define MAX_NAME 1024 #define MAX_FILES 2500 @@ -698,12 +698,15 @@ scan_attrs(hid_t oid, pdcid_t obj_id) void do_attr(hid_t aid, pdcid_t obj_id) { - ssize_t len; - hid_t atype; - hid_t aspace; - char buf[MAX_NAME] = {0}; - char read_buf[TAG_LEN_MAX] = {0}; - pdc_kvtag_t kvtag1; + ssize_t len; + hid_t atype; + hid_t aspace; + char buf[MAX_NAME] = {0}; + char read_buf[TAG_LEN_MAX] = {0}; + // pdc_kvtag_t kvtag1; + char * tag_name; + void * tag_value; + size_t tag_size; /* * Get the name of the attribute. @@ -717,15 +720,15 @@ do_attr(hid_t aid, pdcid_t obj_id) atype = H5Aget_type(aid); H5Aread(aid, atype, read_buf); - kvtag1.name = buf; - kvtag1.value = (void *)read_buf; + tag_name = buf; + tag_value = (void *)read_buf; if (atype == H5T_STRING) { - kvtag1.size = strlen(read_buf) + 1; + tag_size = strlen(read_buf) + 1; } else { - kvtag1.size = H5Tget_size(atype); + tag_size = H5Tget_size(atype); } - PDCobj_put_tag(obj_id, kvtag1.name, kvtag1.value, kvtag1.size); + PDCobj_put_tag(obj_id, tag_name, tag_value, tag_size); /* * Get attribute information: dataspace, data type From 4dbc4a51bed60cfb474c89078dad897a69aaf1d0 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 25 Apr 2023 22:21:29 -0500 Subject: [PATCH 166/216] update pdc_import.c --- tools/pdc_export.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/pdc_export.c b/tools/pdc_export.c index af42e116a..a67f444e3 100644 --- a/tools/pdc_export.c +++ b/tools/pdc_export.c @@ -15,9 +15,9 @@ #endif #include "pdc.h" -#include "pdc_client_server_common.h" -#include "pdc_client_connect.h" -#include "../src/server/include/pdc_server_metadata.h" +// #include "pdc_client_server_common.h" +// #include "pdc_client_connect.h" +// #include "../src/server/include/pdc_server_metadata.h" #include "cjson/cJSON.h" const char *avail_args[] = {"-f"}; From fa745f5aa1c8f36404f3cde97f74f27238f56aac Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 25 Apr 2023 22:22:43 -0500 Subject: [PATCH 167/216] update pdc_export.c --- tools/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 815a081ec..c303a0062 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -90,7 +90,7 @@ add_library(cjson cjson/cJSON.c) set(PROGRAMS pdc_ls pdc_import - pdc_export + # pdc_export ) foreach(program ${PROGRAMS}) From b28957c072e30d3bde6c44becce889bb408ca7a2 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 25 Apr 2023 22:24:17 -0500 Subject: [PATCH 168/216] update pdc_import.c --- tools/pdc_import.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/pdc_import.c b/tools/pdc_import.c index 4a0af982c..1079dd5be 100644 --- a/tools/pdc_import.c +++ b/tools/pdc_import.c @@ -17,6 +17,7 @@ #define MAX_FILES 2500 #define MAX_FILENAME_LEN 64 #define MAX_TAG_SIZE 8192 +#define TAG_LEN_MAX 2048 typedef struct ArrayList { int length; From e2196a98fe032fe81094eb6de9f9bf76f970f4e9 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 25 Apr 2023 22:26:00 -0500 Subject: [PATCH 169/216] update pdc_import.c --- tools/pdc_import.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/pdc_import.c b/tools/pdc_import.c index 1079dd5be..f0598b792 100644 --- a/tools/pdc_import.c +++ b/tools/pdc_import.c @@ -1,6 +1,8 @@ #include #include #include +#include +#include // #define ENABLE_MPI 1 From 9f986d745ddb1f22cf4f37db329142b5340482da Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 25 Apr 2023 22:39:54 -0500 Subject: [PATCH 170/216] update pdc_import.c --- tools/pdc_import.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tools/pdc_import.c b/tools/pdc_import.c index f0598b792..d1a7e785a 100644 --- a/tools/pdc_import.c +++ b/tools/pdc_import.c @@ -83,8 +83,7 @@ int ndset_g = 0; /* FILE *summary_fp_g; */ int max_tag_size_g = 0; pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; -struct timeval write_timer_start_g; -struct timeval write_timer_end_g; +struct timespec write_timer_start_g, write_timer_end_g; struct ArrayList *container_names; int overwrite = 0; @@ -249,9 +248,8 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); #endif - struct timeval pdc_timer_start; - struct timeval pdc_timer_end; - gettimeofday(&pdc_timer_start, 0); + struct timespec pdc_timer_start, pdc_timer_end; + clock_gettime(CLOCK_MONOTONIC, &pdc_timer_start); for (i = 0; i < my_count; i++) { filename = my_filenames[i]; @@ -275,7 +273,8 @@ main(int argc, char **argv) #endif // Checkpoint all metadata after import each hdf5 file if (rank == 0) { - PDC_Client_all_server_checkpoint(); + // FIXME: this should be replaced by a function in public headers. + // PDC_Client_all_server_checkpoint(); } /* printf("%s, %d\n", filename, max_tag_size_g); */ /* printf("\n\n======================\nNumber of datasets: %d\n", ndset_g); */ @@ -288,8 +287,9 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); #endif - gettimeofday(&pdc_timer_end, 0); - double write_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); + clock_gettime(CLOCK_MONOTONIC, &pdc_timer_end); + double write_time = (pdc_timer_end.tv_sec - pdc_timer_start.tv_sec) * 1e9 + + (pdc_timer_end.tv_nsec - pdc_timer_start.tv_nsec); // calculate duration in nanoseconds #ifdef ENABLE_MPI MPI_Reduce(&ndset_g, &total_dset, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); @@ -297,7 +297,7 @@ main(int argc, char **argv) total_dset = ndset_g; #endif if (rank == 0) { - printf("Import %d datasets with %d ranks took %.2f seconds.\n", total_dset, size, write_time); + printf("Import %d datasets with %d ranks took %.2f seconds.\n", total_dset, size, write_time/1e9); } } @@ -553,7 +553,7 @@ do_dset(hid_t did, char *name, char *app_name) obj_region.size = size; if (ndset_g == 1) - gettimeofday(&write_timer_start_g, 0); + clock_gettime(CLOCK_MONOTONIC, &write_timer_start_g); /* PDC_Client_query_metadata_name_timestep(dset_name_g, 0, &meta); */ /* if (meta == NULL) */ @@ -569,12 +569,13 @@ do_dset(hid_t did, char *name, char *app_name) // PDC_Client_write_id(obj_id, &obj_region, buf); if (ndset_g % 100 == 0) { - gettimeofday(&write_timer_end_g, 0); - double elapsed_time = PDC_get_elapsed_time_double(&write_timer_start_g, &write_timer_end_g); - printf("Importer%2d: Finished written 100 objects, took %.2f, my total %d\n", rank, elapsed_time, + clock_gettime(CLOCK_MONOTONIC, &write_timer_end_g); + double elapsed_time = (write_timer_end_g.tv_sec - write_timer_start_g.tv_sec) * 1e9 + + (write_timer_end_g.tv_nsec - write_timer_start_g.tv_nsec); // calculate duration in nanoseconds; + printf("Importer%2d: Finished written 100 objects, took %.2f, my total %d\n", rank, elapsed_time/1e9, ndset_g); fflush(stdout); - gettimeofday(&write_timer_start_g, 0); + clock_gettime(CLOCK_MONOTONIC, &write_timer_start_g); } free(buf); From edd1a1c8d2e88ca755c91bbd2729dcd1876184d1 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 25 Apr 2023 22:41:00 -0500 Subject: [PATCH 171/216] update pdc_import.c --- tools/pdc_import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pdc_import.c b/tools/pdc_import.c index d1a7e785a..88958bb53 100644 --- a/tools/pdc_import.c +++ b/tools/pdc_import.c @@ -542,7 +542,7 @@ do_dset(hid_t did, char *name, char *app_name) scan_attrs(did, obj_id); - pdc_metadata_t *meta = NULL; + // pdc_metadata_t *meta = NULL; obj_region.ndim = ndim; for (i = 0; i < ndim; i++) { offset[i] = 0; From 59a332f49860af3ce98fd7bd81d77977690afa1e Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 25 Apr 2023 22:42:24 -0500 Subject: [PATCH 172/216] update tools/cmake --- tools/CMakeLists.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index c303a0062..e15e21992 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -87,19 +87,19 @@ add_definitions(-DENABLE_MPI=1) add_library(cjson cjson/cJSON.c) -set(PROGRAMS - pdc_ls - pdc_import - # pdc_export - ) +# set(PROGRAMS +# pdc_ls +# pdc_import +# pdc_export +# ) -foreach(program ${PROGRAMS}) - add_executable(${program} ${program}.c) - target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) - target_link_libraries(${program} pdc) - target_link_libraries(${program} cjson) - target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) -endforeach(program) +# foreach(program ${PROGRAMS}) +# add_executable(${program} ${program}.c) +# target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) +# target_link_libraries(${program} pdc) +# target_link_libraries(${program} cjson) +# target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) +# endforeach(program) # Find LibTIFF From d5315f9c1dc3faf44df4b37a02abd2299691cb62 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 25 Apr 2023 23:21:21 -0500 Subject: [PATCH 173/216] clang format --- tools/llsm/csvReader.h | 1 - tools/llsm_importer.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/llsm/csvReader.h b/tools/llsm/csvReader.h index 898f60fdb..d5aa87aaf 100644 --- a/tools/llsm/csvReader.h +++ b/tools/llsm/csvReader.h @@ -54,7 +54,6 @@ void csv_set_escape(char escape); */ void csv_set_newline(char newline); - /** * @brief This function parses a CSV header line and returns a linked list of csv_header_t structs. The header * string may contain quotes and spaces diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 07196d018..d777812c9 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -121,7 +121,7 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds printf("[Rank %4d] Region Transfer for object %s [%d Bytes] Done! Time taken: %.4f seconds\n", rank, - fileName_cell->field_value, image_info->tiff_size , duration / 1e9); + fileName_cell->field_value, image_info->tiff_size, duration / 1e9); // add metadata tags based on the csv row csv_cell_t *cell = fileName_cell; From 17319974b09528837fa530f112bbc58d38a65fa3 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 25 Apr 2023 23:27:38 -0500 Subject: [PATCH 174/216] clang format --- src/tests/kvtag_add_get_benchmark.c | 5 +++-- tools/pdc_import.c | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 6c4457eb4..5cda25433 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -293,7 +293,7 @@ check_and_release_query_result(uint64_t n_query, uint64_t my_obj, uint64_t my_ob } } free(values); - // FIXME: close objects. This is currently commented off to save node hours for benchmarks. + // FIXME: close objects. This is currently commented off to save node hours for benchmarks. // for (i = 0; i < my_obj; i++) { // v = i + my_obj_s; // if (PDCobj_close(obj_ids[i]) < 0) @@ -486,7 +486,8 @@ main(int argc, char *argv[]) free(tag_values); free(obj_ids); - //FIXME: the following is currently commented off to reduce node hours taken by time-consuming resource releasing procedure. + // FIXME: the following is currently commented off to reduce node hours taken by time-consuming resource + // releasing procedure. // closePDC(pdc, cont_prop, cont, obj_prop); done: diff --git a/tools/pdc_import.c b/tools/pdc_import.c index 88958bb53..f51f587c8 100644 --- a/tools/pdc_import.c +++ b/tools/pdc_import.c @@ -288,8 +288,9 @@ main(int argc, char **argv) #endif clock_gettime(CLOCK_MONOTONIC, &pdc_timer_end); - double write_time = (pdc_timer_end.tv_sec - pdc_timer_start.tv_sec) * 1e9 + - (pdc_timer_end.tv_nsec - pdc_timer_start.tv_nsec); // calculate duration in nanoseconds + double write_time = + (pdc_timer_end.tv_sec - pdc_timer_start.tv_sec) * 1e9 + + (pdc_timer_end.tv_nsec - pdc_timer_start.tv_nsec); // calculate duration in nanoseconds #ifdef ENABLE_MPI MPI_Reduce(&ndset_g, &total_dset, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); @@ -297,7 +298,8 @@ main(int argc, char **argv) total_dset = ndset_g; #endif if (rank == 0) { - printf("Import %d datasets with %d ranks took %.2f seconds.\n", total_dset, size, write_time/1e9); + printf("Import %d datasets with %d ranks took %.2f seconds.\n", total_dset, size, + write_time / 1e9); } } @@ -543,7 +545,7 @@ do_dset(hid_t did, char *name, char *app_name) scan_attrs(did, obj_id); // pdc_metadata_t *meta = NULL; - obj_region.ndim = ndim; + obj_region.ndim = ndim; for (i = 0; i < ndim; i++) { offset[i] = 0; size[i] = dims[i]; @@ -570,10 +572,11 @@ do_dset(hid_t did, char *name, char *app_name) // PDC_Client_write_id(obj_id, &obj_region, buf); if (ndset_g % 100 == 0) { clock_gettime(CLOCK_MONOTONIC, &write_timer_end_g); - double elapsed_time = (write_timer_end_g.tv_sec - write_timer_start_g.tv_sec) * 1e9 + - (write_timer_end_g.tv_nsec - write_timer_start_g.tv_nsec); // calculate duration in nanoseconds; - printf("Importer%2d: Finished written 100 objects, took %.2f, my total %d\n", rank, elapsed_time/1e9, - ndset_g); + double elapsed_time = + (write_timer_end_g.tv_sec - write_timer_start_g.tv_sec) * 1e9 + + (write_timer_end_g.tv_nsec - write_timer_start_g.tv_nsec); // calculate duration in nanoseconds; + printf("Importer%2d: Finished written 100 objects, took %.2f, my total %d\n", rank, + elapsed_time / 1e9, ndset_g); fflush(stdout); clock_gettime(CLOCK_MONOTONIC, &write_timer_start_g); } From dc842c56d10ab14c5045af0711399c2ce8f338e7 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Mon, 8 May 2023 11:38:12 -0500 Subject: [PATCH 175/216] Merge tools/llsm_importer (#81) * remove unnecessary install block from CMakeLists.txt * update output * Revert "update output" This reverts commit fe1f8b44995bc0dabd3b957e1032c2da26f56fdd. * build kvtag_add_get_scale * comment off free * update code * 1. kvtag_scale_add_get added \n 2. uint64_t support for obj/tag/query count \n 3. moving work assigning block downwards right before creating objects \n 4. everything is tested working * do while loop added, tested with 1m object and works * 1m objects test works, 10m object test fail as the original also fails * add new executable to test set * enlarge PDC_SERVER_ID_INTERVAL * update code * update console args * add p search test * add console arg for changing number of attributes per object * free allocated memory * fix query count issue * fix attr length definition * code refactored * code refactored * code refactored * code refactored * code refactored * code refactored * fix data type * fix data type * fix data type * add client side statistics * add client side statistics * fix format * clang formatter * update CMake * update CMake * update CMake * free allocated memory properly * clang format * clang format * clang-format-10 * change file name * address review comments * update llsm importer * update llsm importer * update server checkpoint intervals * update gitignore * adding job scripts * adding one debugging msg * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update output for uint64_t * add scripts * update output for uint64_t * update output for uint64_t * update output for uint64_t * update scripts * update scripts * delete debugging message * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * update tag names * update tag names * update query startingpos * update query startingpos * update job scripts * add progressive timing for kvtag_add_get_scale * fix iteration count in final report * update job scripts and benckmark program * update message format * update message format * update message format * update message format * clang format * update job scripts * comment off object/container close procedure in benchmark to save node hours * change the max number of object to 1M * change the max length of attribute value * change the max length of attribute value * llsm tiff import test * llsm tiff import test * llsm tiff import test * llsm tiff import test * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update cmake and llsm_importer * update cmake and llsm_importer * close if in cmake * cmake fix tiff * cmake policy to suppress warning * add pdc include dir * update code * update code * update code * update code * update code * update code * update array generating method * update array generating method * update array generating method * update array generating method * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * fix return type * fix return type * add timing * add timing * fix output * llsm tiff importer 1st version: read csv and import tiff files to PDC, adding metadata available in CSV files and TIFF loader * fix vairable name * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * add scripts * add scripts * add scripts * debugging for nonMPI program * debugging for nonMPI program * debugging for nonMPI program * clang format, without PDC, everything works perfectly. program fails at PDC init stage where PDCprop_create(PDC_CONT_CREATE, pdc) is being created * enable MPI * enable MPI * enlarge BCase size * enlarge BCase size * enlarge BCase size * resolve bcast count * llsm data path in script * llsm data path in script * update csv reader * update csv reader * update csv reader * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * enlarge max write * update pdc * update pdc * update pdc * update pdc * update pdc_import.c * update pdc_import.c * update pdc_export.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update tools/cmake * clang format * clang format --------- Co-authored-by: Houjun Tang --- .../perlmutter/gen_scripts.sh | 2 +- .../perlmutter/template.sh | 11 +- .../perlmutter/template.sh | 9 +- scripts/llsm_importer/clean.sh | 8 + scripts/llsm_importer/gen_script.sh | 21 + scripts/llsm_importer/submit.sh | 76 ++ scripts/llsm_importer/template.sh | 67 ++ src/api/include/pdc_public.h | 4 +- src/server/pdc_client_server_common.c | 9 +- .../pdc_server_region/pdc_server_data.c | 3 +- src/tests/kvtag_add_get_benchmark.c | 16 +- tools/CMakeLists.txt | 86 +- tools/llsm/csvReader.c | 395 +++++++++ tools/llsm/csvReader.h | 190 ++++ tools/llsm/parallelReadTiff.c | 810 ++++++++++++++++++ tools/llsm/parallelReadTiff.h | 34 + tools/llsm/pdc_list.c | 165 ++++ tools/llsm/pdc_list.h | 110 +++ tools/llsm_importer.c | 374 ++++++++ tools/pdc_export.c | 8 +- tools/pdc_import.c | 72 +- 21 files changed, 2405 insertions(+), 65 deletions(-) create mode 100755 scripts/llsm_importer/clean.sh create mode 100755 scripts/llsm_importer/gen_script.sh create mode 100755 scripts/llsm_importer/submit.sh create mode 100755 scripts/llsm_importer/template.sh create mode 100644 tools/llsm/csvReader.c create mode 100644 tools/llsm/csvReader.h create mode 100644 tools/llsm/parallelReadTiff.c create mode 100644 tools/llsm/parallelReadTiff.h create mode 100644 tools/llsm/pdc_list.c create mode 100644 tools/llsm/pdc_list.h create mode 100644 tools/llsm_importer.c diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh b/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh index 62eb1a2b2..1bc285fe7 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh @@ -2,7 +2,7 @@ N_THREAD=NO MAX_NODE=512 MAX_ATTR=1024 -MAX_ATTRLEN=1000 +MAX_ATTRLEN=1000000 for (( i = 1; i <= $MAX_NODE; i*=2 )); do mkdir -p $i diff --git a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh index 9c69b9872..db06915c0 100755 --- a/scripts/kvtag_add_get_benchmark/perlmutter/template.sh +++ b/scripts/kvtag_add_get_benchmark/perlmutter/template.sh @@ -3,7 +3,7 @@ #REGSBATCH -q regular #DBGSBATCH -q debug #SBATCH -N NODENUM -#REGSBATCH -t 4:00:00 +#REGSBATCH -t 1:00:00 #DBGSBATCH -t 0:30:00 #SBATCH -C cpu #SBATCH -J JOBNAME @@ -21,6 +21,7 @@ rm -rf $PDC_TMPDIR/* REPEAT=1 N_NODE=NODENUM +# NCLIENT=127 NCLIENT=31 export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE @@ -35,7 +36,7 @@ CLOSE=$EXECPATH/close_server chmod +x $EXECPATH/* -MAX_OBJ_COUNT=$((1024*1024*1024)) +MAX_OBJ_COUNT=$((1024*1024)) OBJ_INCR=$((MAX_OBJ_COUNT/1024)) ATTR_COUNT=ATTRNUM ATTR_LENGTH=ATTRLEN @@ -48,19 +49,19 @@ echo "" echo "=============" echo "$i Init server" echo "=============" -srun -N $N_NODE -n $N_NODE -c 2 --mem=100000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $SERVER & +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & sleep 5 echo "============================================" echo "KVTAGS with $N_NODE nodes" echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=100000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE echo "" echo "=================" echo "$i Closing server" echo "=================" -srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLOSE +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE date diff --git a/scripts/kvtag_add_get_scale/perlmutter/template.sh b/scripts/kvtag_add_get_scale/perlmutter/template.sh index d48cb29f6..392d498b8 100755 --- a/scripts/kvtag_add_get_scale/perlmutter/template.sh +++ b/scripts/kvtag_add_get_scale/perlmutter/template.sh @@ -3,7 +3,7 @@ #REGSBATCH -q regular #DBGSBATCH -q debug #SBATCH -N NODENUM -#REGSBATCH -t 4:00:00 +#REGSBATCH -t 1:00:00 #DBGSBATCH -t 0:30:00 #SBATCH -C cpu #SBATCH -J JOBNAME @@ -21,6 +21,7 @@ REPEAT=1 N_NODE=NODENUM NCLIENT=31 +# NCLIENT=126 export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE mkdir -p $PDC_TMPDIR @@ -45,19 +46,19 @@ echo "" echo "=============" echo "$i Init server" echo "=============" -srun -N $N_NODE -n $N_NODE -c 2 --mem=128000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $SERVER & +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & sleep 5 echo "============================================" echo "KVTAGS with $N_NODE nodes" echo "============================================" -srun -N $N_NODE -n $TOTALPROC -c 2 --mem=256000 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLIENT $NUM_OBJ $NUM_TAGS $NUM_QUERY +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT $NUM_OBJ $NUM_TAGS $NUM_QUERY echo "" echo "=================" echo "$i Closing server" echo "=================" -srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores stdbuf -i0 -o0 -e0 $CLOSE +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE date diff --git a/scripts/llsm_importer/clean.sh b/scripts/llsm_importer/clean.sh new file mode 100755 index 000000000..d6476962e --- /dev/null +++ b/scripts/llsm_importer/clean.sh @@ -0,0 +1,8 @@ +#!/bin/bash +MAX_NODE=512 + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + + rm -rf $i/* + +done \ No newline at end of file diff --git a/scripts/llsm_importer/gen_script.sh b/scripts/llsm_importer/gen_script.sh new file mode 100755 index 000000000..9d310f2bb --- /dev/null +++ b/scripts/llsm_importer/gen_script.sh @@ -0,0 +1,21 @@ +#!/bin/bash +N_THREAD=NO +MAX_NODE=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +PROG_BASENAME=llsm_importer + +for (( i = 1; i <= $MAX_NODE; i*=2 )); do + mkdir -p $i + JOBNAME=${PROG_BASENAME}_${i} + TARGET=./$i/$JOBNAME.sbatch + cp template.sh $TARGET + sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET + sed -i "s/NODENUM/${i}/g" $TARGET + if [[ "$i" -gt "16" ]]; then + sed -i "s/REG//g" $TARGET + else + sed -i "s/DBG//g" $TARGET + fi +done diff --git a/scripts/llsm_importer/submit.sh b/scripts/llsm_importer/submit.sh new file mode 100755 index 000000000..b9019d149 --- /dev/null +++ b/scripts/llsm_importer/submit.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# MIN_PROC=4 +# MAX_PROC=128 +MIN_PROC=1 +MAX_PROC=512 +MAX_ATTR=1024 +MAX_ATTRLEN=1000 + +PROG_BASENAME=llsm_importer + +curdir=$(pwd) + +first_submit=1 + +for (( i = 1; i <= $MAX_PROC; i*=2 )); do + mkdir -p $i + JOBNAME=${PROG_BASENAME}_${i} + TARGET=./$i/JOBNAME.sh + + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` + echo $njob + while [ $njob -ge 4 ] + do + sleeptime=$[ ( $RANDOM % 1000 ) ] + sleep $sleeptime + njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` + echo $njob + done + + if [[ $first_submit == 1 ]]; then + # Submit first job w/o dependency + echo "Submitting $TARGET" + job=`sbatch $TARGET` + first_submit=0 + else + echo "Submitting $TARGET after ${job: -8}" + job=`sbatch -d afterany:${job: -8} $TARGET` + fi + + sleeptime=$[ ( $RANDOM % 5 ) ] + sleep $sleeptime +done + + +# for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do + +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# while [ $njob -ge 4 ] +# do +# sleeptime=$[ ( $RANDOM % 1000 ) ] +# sleep $sleeptime +# njob=`squeue -u $USER | grep vpic | wc -l` +# echo $njob +# done + + +# cd $curdir/$j +# for filename in ./*.sh ; do + +# if [[ $first_submit == 1 ]]; then +# # Submit first job w/o dependency +# echo "Submitting $filename" +# job=`sbatch $filename` +# first_submit=0 +# else +# echo "Submitting $filename after ${job: -8}" +# job=`sbatch -d afterany:${job: -8} $filename` +# fi + +# sleeptime=$[ ( $RANDOM % 5 ) ] +# sleep $sleeptime + +# done +# done diff --git a/scripts/llsm_importer/template.sh b/scripts/llsm_importer/template.sh new file mode 100755 index 000000000..d736b4076 --- /dev/null +++ b/scripts/llsm_importer/template.sh @@ -0,0 +1,67 @@ +#!/bin/bash -l + +#REGSBATCH -q regular +#DBGSBATCH -q debug +#SBATCH -N NODENUM +#REGSBATCH -t 1:00:00 +#DBGSBATCH -t 0:30:00 +#SBATCH -C cpu +#SBATCH -J JOBNAME +#SBATCH -A m2621 +#SBATCH -o o%j.JOBNAME.out +#SBATCH -e o%j.JOBNAME.out + +# export PDC_DEBUG=0 + + +export PDC_TMPDIR=$SCRATCH/data/pdc/conf + +rm -rf $PDC_TMPDIR/* + +REPEAT=1 + +N_NODE=NODENUM +NCLIENT=1 +# NCLIENT=126 + +export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE +mkdir -p $PDC_TMPDIR + +let TOTALPROC=$NCLIENT*$N_NODE + +EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin +TOOLPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/source/pdc_llsm/tools/build +SERVER=$EXECPATH/pdc_server.exe +CLIENT=$TOOLPATH/llsm_importer +CLOSE=$EXECPATH/close_server + +chmod +x $EXECPATH/* +chmod +x $TOOLPATH/llsm_importer + +LLSM_DATA_PATH=/pscratch/sd/w/wzhang5/data/llsm/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1 +# LLSM_DATA_PATH=/global/cfs/cdirs/m2621/wzhang5/data/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1 +IMGLIST_PATH=${LLSM_DATA_PATH}/ImageList_from_encoder.csv + +date + + +echo "" +echo "=============" +echo "$i Init server" +echo "=============" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & +sleep 5 + + +echo "============================================" +echo "KVTAGS with $N_NODE nodes" +echo "============================================" +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT -f $IMGLIST_PATH + +echo "" +echo "=================" +echo "$i Closing server" +echo "=================" +stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE + +date diff --git a/src/api/include/pdc_public.h b/src/api/include/pdc_public.h index dc286e5e9..8c47976f0 100644 --- a/src/api/include/pdc_public.h +++ b/src/api/include/pdc_public.h @@ -55,7 +55,9 @@ typedef enum { PDC_UINT64 = 9, /* 64-bit unsigned integer types */ PDC_INT16 = 10, PDC_INT8 = 11, - NCLASSES = 12 /* this must be last */ + PDC_UINT8 = 12, + PDC_UINT16 = 13, + NCLASSES = 14 /* this must be last */ } pdc_var_type_t; typedef enum { PDC_PERSIST, PDC_TRANSIENT } pdc_lifetime_t; diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index 289c20527..14417485b 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -265,7 +265,14 @@ PDC_get_var_type_size(pdc_var_type_t dtype) break; case PDC_INT8: ret_value = sizeof(int8_t); - + goto done; + break; + case PDC_UINT8: + ret_value = sizeof(uint8_t); + goto done; + break; + case PDC_UINT16: + ret_value = sizeof(uint16_t); goto done; break; case PDC_INT64: diff --git a/src/server/pdc_server_region/pdc_server_data.c b/src/server/pdc_server_region/pdc_server_data.c index 8388957b0..6f7974b5d 100644 --- a/src/server/pdc_server_region/pdc_server_data.c +++ b/src/server/pdc_server_region/pdc_server_data.c @@ -4695,7 +4695,8 @@ PDC_Server_posix_write(int fd, void *buf, uint64_t write_size) while (write_size > max_write_size) { ret = write(fd, buf, max_write_size); if (ret < 0 || ret != (ssize_t)max_write_size) { - printf("==PDC_SERVER[%d]: write %d failed\n", pdc_server_rank_g, fd); + printf("==PDC_SERVER[%d]: in-loop: write %d failed, ret = %d, max_write_size = %llu\n", + pdc_server_rank_g, fd, ret, max_write_size); ret_value = FAIL; goto done; } diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 7b719ec96..5cda25433 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -293,12 +293,12 @@ check_and_release_query_result(uint64_t n_query, uint64_t my_obj, uint64_t my_ob } } free(values); - // close objects - for (i = 0; i < my_obj; i++) { - v = i + my_obj_s; - if (PDCobj_close(obj_ids[i]) < 0) - printf("fail to close object o%" PRIu64 "\n", v); - } + // FIXME: close objects. This is currently commented off to save node hours for benchmarks. + // for (i = 0; i < my_obj; i++) { + // v = i + my_obj_s; + // if (PDCobj_close(obj_ids[i]) < 0) + // printf("fail to close object o%" PRIu64 "\n", v); + // } } void @@ -486,7 +486,9 @@ main(int argc, char *argv[]) free(tag_values); free(obj_ids); - closePDC(pdc, cont_prop, cont, obj_prop); + // FIXME: the following is currently commented off to reduce node hours taken by time-consuming resource + // releasing procedure. + // closePDC(pdc, cont_prop, cont, obj_prop); done: #ifdef ENABLE_MPI diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index f9915b221..e15e21992 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,5 +1,24 @@ cmake_minimum_required (VERSION 2.8.12) +# Setup cmake policies. +foreach(p + CMP0012 + CMP0013 + CMP0014 + CMP0022 # CMake 2.8.12 + CMP0025 # CMake 3.0 + CMP0053 # CMake 3.1 + CMP0054 # CMake 3.1 + CMP0074 # CMake 3.12 + CMP0075 # CMake 3.12 + CMP0083 # CMake 3.14 + CMP0093 # CMake 3.15 + ) + if(POLICY ${p}) + cmake_policy(SET ${p} NEW) + endif() +endforeach() + project(PDC_VOL C) include_directories( @@ -51,18 +70,65 @@ option(USE_SYSTEM_HDF5 "Use system-installed HDF5." ON) endif() endif() +option(USE_SYSTEM_OPENMP "Use system-installed OpenMP." ON) +if(USE_SYSTEM_OPENMP) + find_package(OpenMP REQUIRED) + if(OPENMP_FOUND) + add_definitions(-DENABLE_OPENMP=1) + set(ENABLE_OPENMP 1) + set(OPENMP_LIBRARIES "${OpenMP_C_LIBRARIES}") + else() + message(FATAL_ERROR "OpenMP not found") + endif() +endif() + + add_definitions(-DENABLE_MPI=1) add_library(cjson cjson/cJSON.c) -set(PROGRAMS - pdc_ls - pdc_import - pdc_export - ) -foreach(program ${PROGRAMS}) - add_executable(${program} ${program}.c) - target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) - target_link_libraries(${program} cjson) -endforeach(program) +# set(PROGRAMS +# pdc_ls +# pdc_import +# pdc_export +# ) + +# foreach(program ${PROGRAMS}) +# add_executable(${program} ${program}.c) +# target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES}) +# target_link_libraries(${program} pdc) +# target_link_libraries(${program} cjson) +# target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR}) +# endforeach(program) + +# Find LibTIFF +option(USE_LIB_TIFF "Enable LibTiff." ON) +if(USE_LIB_TIFF) + find_package(TIFF REQUIRED) + if(TIFF_FOUND) + set(LLSM_LIB_SOURCE + llsm/parallelReadTiff.c + llsm/csvReader.c + llsm/pdc_list.c + ) + # Add the LibTIFF include directory to the include path + include_directories(${TIFF_INCLUDE_DIRS}) + add_library(llsm_tiff ${LLSM_LIB_SOURCE}) + target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS}) + target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES}) + target_link_libraries(llsm_tiff PUBLIC ${TIFF_LIBRARIES}) + target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/llsm) + + + add_executable(llsm_importer llsm_importer.c) + target_link_libraries(llsm_importer ${PDC_EXT_LIB_DEPENDENCIES}) + target_link_libraries(llsm_importer pdc) + target_link_libraries(llsm_importer cjson) + target_link_libraries(llsm_importer ${TIFF_LIBRARIES}) + target_link_libraries(llsm_importer llsm_tiff) + target_include_directories(llsm_importer PUBLIC ${PDC_INCLUDE_DIR}) + else() + message(WARNING "LibTiff not found, ignore building the executables which requires LibTiff support.") + endif() +endif() \ No newline at end of file diff --git a/tools/llsm/csvReader.c b/tools/llsm/csvReader.c new file mode 100644 index 000000000..c6623a287 --- /dev/null +++ b/tools/llsm/csvReader.c @@ -0,0 +1,395 @@ +#include "csvReader.h" + +char csv_delimiter = ','; +char csv_quote = '\"'; +char csv_escape = '\\'; +char csv_newline = '\n'; + +void +csv_set_delimiter(char delimiter) +{ + csv_delimiter = delimiter; +} + +void +csv_set_quote(char quote) +{ + csv_quote = quote; +} + +void +csv_set_escape(char escape) +{ + csv_escape = escape; +} + +void +csv_set_newline(char newline) +{ + csv_newline = newline; +} + +csv_header_t * +csv_parse_header(char *line, char *field_types) +{ + csv_header_t *first_header = NULL; + csv_header_t *last_header = NULL; + char * token = NULL; + char * saveptr = NULL; + int field_index = 0; + int in_quotes = 0; + int value_start = 0; + int i = 0; + + for (int i = 0; line[i] != csv_newline; ++i) { + if (line[i] == csv_quote) { + in_quotes = !in_quotes; + } + else if (!in_quotes && (line[i] == csv_delimiter || line[i + 1] == csv_newline)) { + // Allocate memory for the header struct + csv_header_t *header = (csv_header_t *)malloc(sizeof(csv_header_t)); + if (header == NULL) { + return NULL; + } + // Remove quotes and spaces from the field name + header->field_name = strndup(line + value_start, i - value_start + (line[i + 1] == csv_newline)); + + // Set the field index + header->field_index = field_index; + + // Set the field type + if (field_types != NULL) { + header->field_type = field_types[field_index]; + } + else { + header->field_type = 's'; + } + + // Set the next pointer to NULL + header->next = NULL; + + // Add the header to the linked list + if (first_header == NULL) { + first_header = header; + last_header = header; + } + else { + last_header->next = header; + last_header = header; + } + + value_start = i + 1; + field_index++; + } + } + + return first_header; +} + +csv_row_t * +csv_parse_row(char *line, csv_header_t *header) +{ + csv_cell_t * first_cell = NULL; + csv_cell_t * last_cell = NULL; + csv_header_t *current_header = header; + char * token = NULL; + char * saveptr = NULL; + int field_index = 0; + int in_quotes = 0; + int value_start = 0; + int i = 0; + + for (int i = 0; line[i] != csv_newline; ++i) { + if (line[i] == csv_quote) { + in_quotes = !in_quotes; + } + else if (!in_quotes && (line[i] == csv_delimiter || line[i + 1] == csv_newline)) { + // Allocate memory for the cell struct + csv_cell_t *cell = (csv_cell_t *)malloc(sizeof(csv_cell_t)); + if (cell == NULL) { + return NULL; + } + + // Set the field name + cell->header = current_header; + + // Set the field value + cell->field_value = strndup(line + value_start, i - value_start + (line[i + 1] == csv_newline)); + + // Set the next pointer to NULL + cell->next = NULL; + + // Add the cell to the linked list + if (first_cell == NULL) { + first_cell = cell; + last_cell = cell; + } + else { + last_cell->next = cell; + last_cell = cell; + } + + value_start = i + 1; + field_index++; + current_header = current_header->next; + } + } + csv_row_t *row = (csv_row_t *)malloc(sizeof(csv_row_t)); + row->first_cell = first_cell; + row->next = NULL; + return row; +} + +csv_cell_t * +csv_get_field_value_by_name(csv_row_t *row, csv_header_t *header, char *field_name) +{ + csv_cell_t *cell = row->first_cell; + while (cell != NULL) { + if (strcmp(cell->header->field_name, field_name) == 0) { + return cell; + } + cell = cell->next; + } + return NULL; +} + +csv_cell_t * +csv_get_field_value_by_index(csv_row_t *row, csv_header_t *header, int field_index) +{ + csv_cell_t *cell = row->first_cell; + while (cell != NULL) { + if (cell->header->field_index == field_index) { + return cell; + } + cell = cell->next; + } + return NULL; +} + +csv_table_t * +csv_parse_file(char *file_name, char *field_types) +{ + FILE *fp = fopen(file_name, "r"); + if (fp == NULL) { + return NULL; + } + + // Allocate memory for the table struct + csv_table_t *table = (csv_table_t *)malloc(sizeof(csv_table_t)); + if (table == NULL) { + return NULL; + } + + // Read the first line of the file + char * line = NULL; + size_t len = 0; + ssize_t read = getline(&line, &len, fp); + + // Parse the header + table->first_header = csv_parse_header(line, field_types); + + // Parse the rows + csv_row_t *first_row = NULL; + csv_row_t *last_row = NULL; + while ((read = getline(&line, &len, fp)) != -1) { + // Allocate memory for the row struct + csv_row_t *row = csv_parse_row(line, table->first_header); + if (row == NULL) { + return NULL; + } + + // Add the row to the linked list + if (first_row == NULL) { + first_row = row; + last_row = row; + } + else { + last_row->next = row; + last_row = row; + } + } + + table->first_row = first_row; + + return table; +} + +csv_table_t * +csv_parse_list(PDC_LIST *list, char *field_types) +{ + csv_table_t *table = (csv_table_t *)malloc(sizeof(csv_table_t)); + if (table == NULL) { + return NULL; + } + int num_file_read = 0; + csv_row_t *first_row = NULL; + csv_row_t *last_row = NULL; + + PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); + while (pdc_list_iterator_has_next(iter)) { + char *line = (char *)pdc_list_iterator_next(iter); + if (num_file_read == 0) { + table->first_header = csv_parse_header(line, field_types); + } + else { + // Allocate memory for the row struct + csv_row_t *row = csv_parse_row(line, table->first_header); + if (row == NULL) { + return NULL; + } + + // Add the row to the linked list + if (first_row == NULL) { + first_row = row; + last_row = row; + } + else { + last_row->next = row; + last_row = row; + } + } + num_file_read++; + } + + table->first_row = first_row; + + return table; +} + +void +csv_free_header(csv_header_t *header) +{ + csv_header_t *current_header = header; + csv_header_t *next_header = NULL; + while (current_header != NULL) { + next_header = current_header->next; + free(current_header->field_name); + free(current_header); + current_header = next_header; + } +} + +void +csv_free_row(csv_row_t *row) +{ + csv_row_t *current_row = row; + csv_row_t *next_row = NULL; + while (current_row != NULL) { + next_row = current_row->next; + csv_free_cell(current_row->first_cell); + free(current_row); + current_row = next_row; + } +} + +void +csv_free_cell(csv_cell_t *cell) +{ + csv_cell_t *current_cell = cell; + csv_cell_t *next_cell = NULL; + while (current_cell != NULL) { + next_cell = current_cell->next; + free(current_cell->field_value); + free(current_cell); + current_cell = next_cell; + } +} + +void +csv_free_table(csv_table_t *table) +{ + csv_free_header(table->first_header); + csv_free_row(table->first_row); + free(table); +} + +void +csv_print_header(csv_header_t *header) +{ + csv_header_t *current_header = header; + while (current_header != NULL) { + printf("%s", current_header->field_name); + if (current_header->next != NULL) { + printf(", "); + } + current_header = current_header->next; + } + printf("\n"); +} + +void +csv_print_row(csv_row_t *row, int with_key) +{ + csv_cell_t *current_cell = row->first_cell; + while (current_cell != NULL) { + csv_print_cell(current_cell, with_key); + if (current_cell->next != NULL) { + printf(", "); + } + if (with_key) { + printf("\n"); + } + current_cell = current_cell->next; + } + printf("\n"); +} + +void +csv_print_cell(csv_cell_t *cell, int with_key) +{ + if (with_key) { + printf("%s: ", cell->header->field_name); + } + switch (cell->header->field_type) { + case 'i': + printf("%ld", strtol(cell->field_value, NULL, 10)); + break; + + case 'f': + printf("%f", strtod(cell->field_value, NULL)); + break; + + case 's': + printf("%s", cell->field_value); + break; + + default: + printf("%s", cell->field_value); + break; + } +} + +void +csv_print_table(csv_table_t *table) +{ + csv_print_header(table->first_header); + csv_row_t *current_row = table->first_row; + while (current_row != NULL) { + csv_print_row(current_row, 0); + current_row = current_row->next; + } +} + +int +csv_get_num_rows(csv_table_t *table) +{ + int num_rows = 0; + csv_row_t *current_row = table->first_row; + while (current_row != NULL) { + num_rows++; + current_row = current_row->next; + } + return num_rows; +} + +int +csv_get_num_fields(csv_table_t *table) +{ + int num_fields = 0; + csv_header_t *current_header = table->first_header; + while (current_header != NULL) { + num_fields++; + current_header = current_header->next; + } + return num_fields; +} \ No newline at end of file diff --git a/tools/llsm/csvReader.h b/tools/llsm/csvReader.h new file mode 100644 index 000000000..d5aa87aaf --- /dev/null +++ b/tools/llsm/csvReader.h @@ -0,0 +1,190 @@ +#ifndef CSVREADER_H +#define CSVREADER_H + +#include +#include +#include + +#include "pdc_list.h" + +typedef struct csv_header_t { + char * field_name; + int field_index; + char field_type; + struct csv_header_t *next; +} csv_header_t; + +typedef struct csv_cell_t { + char * field_value; + csv_header_t * header; + struct csv_cell_t *next; +} csv_cell_t; + +typedef struct csv_row_t { + csv_cell_t * first_cell; + struct csv_row_t *next; +} csv_row_t; + +typedef struct csv_table_t { + csv_header_t *first_header; + csv_row_t * first_row; +} csv_table_t; + +/** + * @brief This function sets the delimiter for the CSV file. The default is a comma. + * @param delimiter The delimiter to use. + */ +void csv_set_delimiter(char delimiter); + +/** + * @brief This function sets the quote character for the CSV file. The default is a double quote. + * @param quote The quote character to use. + */ +void csv_set_quote(char quote); + +/** + * @brief This function sets the escape character for the CSV file. The default is a backslash. + * @param escape The escape character to use. + */ +void csv_set_escape(char escape); + +/** + * @brief This function sets the newline character for the CSV file. The default is a newline. + * @param newline The newline character to use. + */ +void csv_set_newline(char newline); + +/** + * @brief This function parses a CSV header line and returns a linked list of csv_header_t structs. The header + * string may contain quotes and spaces + * @param line The CSV header line to parse. + * @param field_types A string of field types. The field types are 's' for string, 'i' for long integer, 'f' + * for float, and 'd' for double. If this is NULL, all fields are assumed to be strings. + * + * @return A pointer to the first csv_header_t struct in the linked list. + */ +csv_header_t *csv_parse_header(char *line, char *field_types); + +/** + * @brief This function parse a CSV row line and returns a linked list of csv_cell_t structs. The row string + * may contain quotes and spaces + * @param line The CSV row line to parse. + * @param header A pointer to the first csv_header_t struct in the linked list. + * + * @return A pointer to the csv_row_t struct. The value in the csv_cell should be + * free of quotes or spaces. + */ +csv_row_t *csv_parse_row(char *line, csv_header_t *header); + +/** + * @brief This function returns the string value of a field for a given row string. The row string may contain + * quotes and spaces + * @param row The CSV row to look for. + * @param header A pointer to the first csv_header_t struct in the linked list. + * @param field_name The name of the field to get the value for. + * + * @return A pointer to the csv_cell struct of the field. The value in the csv_cell should be free of quotes + * or spaces. + */ +csv_cell_t *csv_get_field_value_by_name(csv_row_t *row, csv_header_t *header, char *field_name); + +/** + * @brief This function returns the string value of a field for a given row string. The row string may contain + * quotes and spaces + * @param row The CSV row to look for. + * @param header A pointer to the first csv_header_t struct in the linked list. + * @param field_index The index of the field to get the value for. + * + * @return A pointer to the csv_cell struct of the field. The value in the csv_cell should be free of quotes + * or spaces. + */ +csv_cell_t *csv_get_field_value_by_index(csv_row_t *row, csv_header_t *header, int field_index); + +/** + * @brief This function parses a CSV file and returns a csv_table_t struct. + * @param file_name The name of the CSV file to parse. + * @param field_types A string of field types. The field types are 's' for string, 'i' for long integer, 'f' + * for float, and 'd' for double. If this is NULL, all fields are assumed to be strings. + * + * @return A pointer to the csv_table_t struct. + */ +csv_table_t *csv_parse_file(char *file_name, char *field_types); + +/** + * @brief This function parses a PDC_LIST of strings as a CSV file and returns a csv_table_t struct. + * @param list A PDC_LIST of strings to parse. + * @param field_types A string of field types. The field types are 's' for string, 'i' for long integer, 'f' + * for float, and 'd' for double. If this is NULL, all fields are assumed to be strings. + * + * @return A pointer to the csv_table_t struct. + */ +csv_table_t *csv_parse_list(PDC_LIST *list, char *field_types); + +/** + * @brief This function frees the memory allocated for a csv_table_t struct. + * @param table A pointer to the csv_table_t struct to free. + */ +void csv_free_table(csv_table_t *table); + +/** + * @brief This function frees the memory allocated for a csv_header_t struct. + * @param header A pointer to the csv_header_t struct to free. + */ +void csv_free_header(csv_header_t *header); + +/** + * @brief This function frees the memory allocated for a csv_row_t struct. + * @param row A pointer to the csv_row_t struct to free. + */ +void csv_free_row(csv_row_t *row); + +/** + * @brief This function frees the memory allocated for a csv_cell_t struct. + * @param cell A pointer to the csv_cell_t struct to free. + */ +void csv_free_cell(csv_cell_t *cell); + +/** + * @brief This function prints the contents of a csv_table_t struct. + * @param table A pointer to the csv_table_t struct to print. + */ +void csv_print_table(csv_table_t *table); + +/** + * @brief This function prints the contents of a csv_header_t struct. + * @param header A pointer to the csv_header_t struct to print. + */ +void csv_print_header(csv_header_t *header); + +/** + * @brief This function prints the contents of a csv_row_t struct. + * @param row A pointer to the csv_row_t struct to print. + * @param with_key A flag to indicate whether to print the key or not. + */ + +void csv_print_row(csv_row_t *row, int with_key); + +/** + * @brief This function prints the contents of a csv_cell_t struct. + * @param cell A pointer to the csv_cell_t struct to print. + * @param with_key A flag to indicate whether to print the key or not. + */ +void csv_print_cell(csv_cell_t *cell, int with_key); + +/** + * @brief This function returns the number of rows in a csv_table_t struct. + * @param table A pointer to the csv_table_t struct. + * + * @return The number of rows in the table. + */ +int csv_get_num_rows(csv_table_t *table); + +/** + * @brief This function returns the number of fields in a csv_table_t struct. + * @param table A pointer to the csv_table_t struct. + * + * @return The number of fields in the table. + */ +int csv_get_num_fields(csv_table_t *table); + +#endif // CSVREADER_H \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c new file mode 100644 index 000000000..4da885e61 --- /dev/null +++ b/tools/llsm/parallelReadTiff.c @@ -0,0 +1,810 @@ +#include "parallelReadTiff.h" +#include "tiffio.h" + +// #define ENABLE_OPENMP + +#ifdef ENABLE_OPENMP +#include "omp.h" +#endif + +#define CREATE_ARRAY(result_var, type, ndim, dim) \ + do { \ + size_t i = 0, dim_prod = 1; \ + for (i = 0; i < (ndim); i++) { \ + dim_prod *= (dim)[i]; \ + } \ + result_var = (void *)malloc(dim_prod * sizeof(type)); \ + } while (0) + +void +DummyHandler(const char *module, const char *fmt, va_list ap) +{ + // ignore errors and warnings +} + +// Backup method in case there are errors reading strips +void +readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (z - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; + + int32_t w; +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (w = 0; w < numWorkers; w++) { + + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:threadError | Thread %d: File \"%s\" cannot be opened\n", w, fileName); + + void *buffer = malloc(x * bytes); + for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); dir++) { + if (dir >= z + startSlice) + break; + + int counter = 0; + while (!TIFFSetDirectory(tif, (uint64_t)dir) && counter < 3) { + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, dir, + counter + 1); + counter++; + } + + for (int64_t i = 0; i < y; i++) { + TIFFReadScanline(tif, buffer, i, 0); + if (!flipXY) { + memcpy(tiff + ((i * x) * bytes), buffer, x * bytes); + continue; + } + // loading the data into a buffer + switch (bits) { + case 8: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint8_t *)buffer)[j]; + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint16_t *)buffer)[j]; + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((float *)buffer)[j]; + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((double *)buffer)[j]; + } + break; + } + } + } + free(buffer); + TIFFClose(tif); + } +} + +void +readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (z - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; + + uint16_t compressed = 1; + TIFF * tif = TIFFOpen(fileName, "r"); + TIFFGetField(tif, TIFFTAG_COMPRESSION, &compressed); + + int32_t w; + uint8_t errBak = 0; + uint8_t err = 0; + char errString[10000]; + if (compressed > 1 || z < 32768) { + TIFFClose(tif); +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (w = 0; w < numWorkers; w++) { + + uint8_t outCounter = 0; + TIFF * tif = TIFFOpen(fileName, "r"); + while (!tif) { + tif = TIFFOpen(fileName, "r"); + if (outCounter == 3) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + err = 1; + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); + } + continue; + } + outCounter++; + } + + void *buffer = malloc(x * stripSize * bytes); + for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); + dir++) { + if (dir >= z + startSlice || err) + break; + + uint8_t counter = 0; + while (!TIFFSetDirectory(tif, (uint64_t)dir) && counter < 3) { + counter++; + if (counter == 3) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + err = 1; + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); + } + } + } + if (err) + break; + for (int64_t i = 0; i * stripSize < y; i++) { + + // loading the data into a buffer + int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize * x * bytes); + if (cBytes < 0) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + errBak = 1; + err = 1; + sprintf(errString, "Thread %d: Strip %ld cannot be read\n", w, i); + } + break; + } + if (!flipXY) { + memcpy(tiff + ((i * stripSize * x) * bytes), buffer, cBytes); + continue; + } + switch (bits) { + case 8: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((uint8_t *)buffer)[j + (k * x)]; + } + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((uint16_t *)buffer)[j + (k * x)]; + } + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((float *)buffer)[j + (k * x)]; + } + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + (k + (i * stripSize))) + + ((dir - startSlice) * (x * y))] = + ((double *)buffer)[j + (k * x)]; + } + } + break; + } + } + } + free(buffer); + TIFFClose(tif); + } + } + else { + uint64_t stripsPerDir = (uint64_t)ceil((double)y / (double)stripSize); +#ifdef _WIN32 + int fd = open(fileName, O_RDONLY | O_BINARY); +#else + int fd = open(fileName, O_RDONLY); +#endif + if (fd == -1) + printf("disk:threadError | File \"%s\" cannot be opened from Disk\n", fileName); + + if (!tif) + printf("tiff:threadError | File \"%s\" cannot be opened\n", fileName); + uint64_t offset = 0; + uint64_t *offsets = NULL; + TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); + uint64_t *byteCounts = NULL; + TIFFGetField(tif, TIFFTAG_STRIPBYTECOUNTS, &byteCounts); + if (!offsets || !byteCounts) + printf("tiff:threadError | Could not get offsets or byte counts from the tiff file\n"); + offset = offsets[0]; + uint64_t fOffset = offsets[stripsPerDir - 1] + byteCounts[stripsPerDir - 1]; + uint64_t zSize = fOffset - offset; + TIFFSetDirectory(tif, 1); + TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); + uint64_t gap = offsets[0] - fOffset; + + lseek(fd, offset, SEEK_SET); + + TIFFClose(tif); + uint64_t curr = 0; + uint64_t bytesRead = 0; + // TESTING + // Not sure if we will need to read in chunks like for ImageJ + for (uint64_t i = 0; i < z; i++) { + bytesRead = read(fd, tiff + curr, zSize); + curr += bytesRead; + lseek(fd, gap, SEEK_CUR); + } + close(fd); + uint64_t size = x * y * z * (bits / 8); + void * tiffC = malloc(size); + memcpy(tiffC, tiff, size); +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (uint64_t k = 0; k < z; k++) { + for (uint64_t j = 0; j < x; j++) { + for (uint64_t i = 0; i < y; i++) { + switch (bits) { + case 8: + ((uint8_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint8_t *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 16: + ((uint16_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint16_t *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 32: + ((float *)tiff)[i + (j * y) + (k * x * y)] = + ((float *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 64: + ((double *)tiff)[i + (j * y) + (k * x * y)] = + ((double *)tiffC)[j + (i * x) + (k * x * y)]; + break; + } + } + } + } + free(tiffC); + } + if (err) { + if (errBak) + readTiffParallelBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); + else + printf("tiff:threadError %s\n", errString); + } +} + +// Backup method in case there are errors reading strips +void +readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + int32_t batchSize = (y - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; + + int32_t w; +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (w = 0; w < numWorkers; w++) { + + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:threadError | Thread %d: File \"%s\" cannot be opened\n", w, fileName); + + void *buffer = malloc(x * bytes); + for (int64_t dir = startSlice + (w * batchSize); dir < startSlice + ((w + 1) * batchSize); dir++) { + if (dir >= z + startSlice) + break; + + int counter = 0; + while (!TIFFSetDirectory(tif, (uint64_t)0) && counter < 3) { + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, dir, + counter + 1); + counter++; + } + + for (int64_t i = (w * batchSize); i < ((w + 1) * batchSize); i++) { + if (i >= y) + break; + TIFFReadScanline(tif, buffer, i, 0); + if (!flipXY) { + memcpy(tiff + ((i * x) * bytes), buffer, x * bytes); + continue; + } + // loading the data into a buffer + switch (bits) { + case 8: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint8_t *)buffer)[j]; + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((uint16_t *)buffer)[j]; + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((float *)buffer)[j]; + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + i) + ((dir - startSlice) * (x * y))] = + ((double *)buffer)[j]; + } + break; + } + } + } + free(buffer); + TIFFClose(tif); + } +} + +void +readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY) +{ + int32_t numWorkers = omp_get_max_threads(); + uint64_t stripsPerDir = (uint64_t)ceil((double)y / (double)stripSize); + int32_t batchSize = (stripsPerDir - 1) / numWorkers + 1; + uint64_t bytes = bits / 8; + + int32_t w; + uint8_t err = 0; + uint8_t errBak = 0; + char errString[10000]; + +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (w = 0; w < numWorkers; w++) { + + uint8_t outCounter = 0; + TIFF * tif = TIFFOpen(fileName, "r"); + while (!tif) { + tif = TIFFOpen(fileName, "r"); + if (outCounter == 3) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + err = 1; + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); + } + continue; + } + outCounter++; + } + + void *buffer = malloc(x * stripSize * bytes); + + uint8_t counter = 0; + while (!TIFFSetDirectory(tif, 0) && counter < 3) { + printf("Thread %d: File \"%s\" Directory \"%d\" failed to open. Try %d\n", w, fileName, 0, + counter + 1); + counter++; + if (counter == 3) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + err = 1; + sprintf(errString, "Thread %d: File \"%s\" cannot be opened\n", w, fileName); + } + } + } + for (int64_t i = (w * batchSize); i < (w + 1) * batchSize; i++) { + if (i * stripSize >= y || err) + break; + // loading the data into a buffer + int64_t cBytes = TIFFReadEncodedStrip(tif, i, buffer, stripSize * x * bytes); + if (cBytes < 0) { +#ifdef ENABLE_OPENMP +#pragma omp critical +#endif + { + errBak = 1; + err = 1; + sprintf(errString, "Thread %d: Strip %ld cannot be read\n", w, i); + } + break; + } + if (!flipXY) { + memcpy(tiff + ((i * stripSize * x) * bytes), buffer, cBytes); + continue; + } + switch (bits) { + case 8: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint8_t *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((uint8_t *)buffer)[j + (k * x)]; + } + } + break; + case 16: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((uint16_t *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((uint16_t *)buffer)[j + (k * x)]; + } + } + break; + case 32: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((float *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((float *)buffer)[j + (k * x)]; + } + } + break; + case 64: + // Map Values to flip x and y for MATLAB + for (int64_t k = 0; k < stripSize; k++) { + if ((k + (i * stripSize)) >= y) + break; + for (int64_t j = 0; j < x; j++) { + ((double *)tiff)[((j * y) + (k + (i * stripSize)))] = + ((double *)buffer)[j + (k * x)]; + } + } + break; + } + } + free(buffer); + TIFFClose(tif); + } + + if (err) { + if (errBak) + readTiffParallel2DBak(x, y, z, fileName, tiff, bits, startSlice, flipXY); + else + printf("tiff:threadError %s\n", errString); + } +} + +// Reading images saved by ImageJ +void +readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void *tiff, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY) +{ +#ifdef _WIN32 + int fd = open(fileName, O_RDONLY | O_BINARY); +#else + int fd = open(fileName, O_RDONLY); +#endif + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:threadError | File \"%s\" cannot be opened\n", fileName); + uint64_t offset = 0; + uint64_t *offsets = NULL; + TIFFGetField(tif, TIFFTAG_STRIPOFFSETS, &offsets); + if (offsets) + offset = offsets[0]; + + TIFFClose(tif); + lseek(fd, offset, SEEK_SET); + uint64_t bytes = bits / 8; + //#pragma omp parallel for + /* + for(uint64_t i = 0; i < z; i++){ + uint64_t cOffset = x*y*bytes*i; + //pread(fd,tiff+cOffset,x*y*bytes,offset+cOffset); + read(fd,tiff+cOffset,x*y*bytes); + }*/ + uint64_t chunk = 0; + uint64_t tBytes = x * y * z * bytes; + uint64_t bytesRead; + uint64_t rBytes = tBytes; + if (tBytes < INT_MAX) + bytesRead = read(fd, tiff, tBytes); + else { + while (chunk < tBytes) { + rBytes = tBytes - chunk; + if (rBytes > INT_MAX) + bytesRead = read(fd, tiff + chunk, INT_MAX); + else + bytesRead = read(fd, tiff + chunk, rBytes); + chunk += bytesRead; + } + } + close(fd); + // Swap endianess for types greater than 8 bits + // TODO: May need to change later because we may not always need to swap + if (bits > 8) { +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (uint64_t i = 0; i < x * y * z; i++) { + switch (bits) { + case 16: + //((uint16_t*)tiff)[i] = ((((uint16_t*)tiff)[i] & 0xff) >> 8) | (((uint16_t*)tiff)[i] << + // 8); + //((uint16_t*)tiff)[i] = bswap_16(((uint16_t*)tiff)[i]); + ((uint16_t *)tiff)[i] = + ((((uint16_t *)tiff)[i] << 8) & 0xff00) | ((((uint16_t *)tiff)[i] >> 8) & 0x00ff); + break; + case 32: + //((num & 0xff000000) >> 24) | ((num & 0x00ff0000) >> 8) | ((num & 0x0000ff00) << 8) | + //(num << 24) + //((float*)tiff)[i] = bswap_32(((float*)tiff)[i]); + ((uint32_t *)tiff)[i] = ((((uint32_t *)tiff)[i] << 24) & 0xff000000) | + ((((uint32_t *)tiff)[i] << 8) & 0x00ff0000) | + ((((uint32_t *)tiff)[i] >> 8) & 0x0000ff00) | + ((((uint32_t *)tiff)[i] >> 24) & 0x000000ff); + break; + case 64: + //((double*)tiff)[i] = bswap_64(((double*)tiff)[i]); + ((uint64_t *)tiff)[i] = ((((uint64_t *)tiff)[i] << 56) & 0xff00000000000000UL) | + ((((uint64_t *)tiff)[i] << 40) & 0x00ff000000000000UL) | + ((((uint64_t *)tiff)[i] << 24) & 0x0000ff0000000000UL) | + ((((uint64_t *)tiff)[i] << 8) & 0x000000ff00000000UL) | + ((((uint64_t *)tiff)[i] >> 8) & 0x00000000ff000000UL) | + ((((uint64_t *)tiff)[i] >> 24) & 0x0000000000ff0000UL) | + ((((uint64_t *)tiff)[i] >> 40) & 0x000000000000ff00UL) | + ((((uint64_t *)tiff)[i] >> 56) & 0x00000000000000ffUL); + break; + } + } + } + // Find a way to do this in-place without making a copy + if (flipXY) { + uint64_t size = x * y * z * (bits / 8); + void * tiffC = malloc(size); + memcpy(tiffC, tiff, size); +#ifdef ENABLE_OPENMP +#pragma omp parallel for +#endif + for (uint64_t k = 0; k < z; k++) { + for (uint64_t j = 0; j < x; j++) { + for (uint64_t i = 0; i < y; i++) { + switch (bits) { + case 8: + ((uint8_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint8_t *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 16: + ((uint16_t *)tiff)[i + (j * y) + (k * x * y)] = + ((uint16_t *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 32: + ((float *)tiff)[i + (j * y) + (k * x * y)] = + ((float *)tiffC)[j + (i * x) + (k * x * y)]; + break; + case 64: + ((double *)tiff)[i + (j * y) + (k * x * y)] = + ((double *)tiffC)[j + (i * x) + (k * x * y)]; + break; + } + } + } + } + free(tiffC); + } +} + +uint8_t +isImageJIm(TIFF *tif) +{ + if (!tif) + return 0; + char *tiffDesc = NULL; + if (TIFFGetField(tif, TIFFTAG_IMAGEDESCRIPTION, &tiffDesc)) { + if (strstr(tiffDesc, "ImageJ")) { + return 1; + } + } + return 0; +} + +uint64_t +imageJImGetZ(TIFF *tif) +{ + if (!tif) + return 0; + char *tiffDesc = NULL; + if (TIFFGetField(tif, TIFFTAG_IMAGEDESCRIPTION, &tiffDesc)) { + if (strstr(tiffDesc, "ImageJ")) { + char *nZ = strstr(tiffDesc, "images="); + if (nZ) { + nZ += 7; + char *temp; + return strtol(nZ, &temp, 10); + } + } + } + return 0; +} + +void +get_tiff_info(char *fileName, parallel_tiff_range_t *strip_range, uint64_t *x, uint64_t *y, uint64_t *z, + uint64_t *bits, uint64_t *startSlice, uint64_t *stripSize, uint64_t *is_imageJ, + uint64_t *imageJ_Z) +{ + TIFFSetWarningHandler(DummyHandler); + TIFF *tif = TIFFOpen(fileName, "r"); + if (!tif) + printf("tiff:inputError | File \"%s\" cannot be opened", fileName); + + TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, x); + TIFFGetField(tif, TIFFTAG_IMAGELENGTH, y); + + if (strip_range == NULL) { + uint16_t s = 0, m = 0, t = 1; + while (TIFFSetDirectory(tif, t)) { + s = t; + t *= 8; + if (s > t) { + t = 65535; + printf("Number of slices > 32768\n"); + break; + } + } + while (s != t) { + m = (s + t + 1) / 2; + if (TIFFSetDirectory(tif, m)) { + s = m; + } + else { + if (m > 0) + t = m - 1; + else + t = m; + } + } + *z = s + 1; + } + else { + if (strip_range->length != 2) { + printf("tiff:inputError | Input range is not 2"); + } + else { + *startSlice = (uint64_t)(*(strip_range->range)) - 1; + *z = (uint64_t)(*(strip_range->range + 1)) - startSlice[0]; + if (!TIFFSetDirectory(tif, startSlice[0] + z[0] - 1) || !TIFFSetDirectory(tif, startSlice[0])) { + printf("tiff:rangeOutOfBound | Range is out of bounds"); + } + } + } + + *is_imageJ = isImageJIm(tif); + *imageJ_Z = imageJImGetZ(tif); + if (*is_imageJ) { + *is_imageJ = 1; + *imageJ_Z = imageJImGetZ(tif); + if (*imageJ_Z) + *z = *imageJ_Z; + } + + TIFFGetField(tif, TIFFTAG_BITSPERSAMPLE, bits); + TIFFGetField(tif, TIFFTAG_ROWSPERSTRIP, stripSize); + TIFFClose(tif); +} + +void * +_get_tiff_array(int bits, int ndim, size_t *dims) +{ + void *tiff = NULL; + if (bits == 8) { + CREATE_ARRAY(tiff, uint8_t, ndim, dims); + } + else if (bits == 16) { + CREATE_ARRAY(tiff, uint16_t, ndim, dims); + } + else if (bits == 32) { + CREATE_ARRAY(tiff, float, ndim, dims); + } + else if (bits == 64) { + CREATE_ARRAY(tiff, double, ndim, dims); + } + return tiff; +} + +void +_TIFF_load(char *fileName, uint8_t isImageJIm, uint64_t x, uint64_t y, uint64_t z, uint64_t bits, + uint64_t startSlice, uint64_t stripSize, uint8_t flipXY, int ndim, size_t *dims, void **tiff_ptr) +{ + if (tiff_ptr == NULL) { + printf("tiff:dataTypeError, Data type not suppported\n"); + } + *tiff_ptr = _get_tiff_array(bits, ndim, dims); + // Case for ImageJ + if (isImageJIm) { + readTiffParallelImageJ(x, y, z, fileName, *tiff_ptr, bits, startSlice, stripSize, flipXY); + } + // Case for 2D + else if (z <= 1) { + readTiffParallel2D(x, y, z, fileName, *tiff_ptr, bits, startSlice, stripSize, flipXY); + } + // Case for 3D + else { + readTiffParallel(x, y, z, fileName, *tiff_ptr, bits, startSlice, stripSize, flipXY); + } +} + +void +parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, + image_info_t **image_info) +{ + uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 0, is_imageJ = 0, imageJ_Z = 0; + + get_tiff_info(fileName, strip_range, &x, &y, &z, &bits, &startSlice, &stripeSize, &is_imageJ, &imageJ_Z); + + int ndim = 3; + uint64_t dims[ndim]; + dims[0] = flipXY ? y : x; + dims[1] = flipXY ? x : y; + dims[2] = z; + + *image_info = (image_info_t *)malloc(sizeof(image_info_t)); + (*image_info)->x = dims[0]; + (*image_info)->y = dims[1]; + (*image_info)->z = dims[2]; + (*image_info)->bits = bits; + (*image_info)->startSlice = startSlice; + (*image_info)->stripeSize = stripeSize; + (*image_info)->is_imageJ = is_imageJ; + (*image_info)->imageJ_Z = imageJ_Z; + (*image_info)->tiff_size = dims[0] * dims[1] * dims[2] * (bits / 8); + + _TIFF_load(fileName, is_imageJ, x, y, z, bits, startSlice, stripeSize, flipXY, ndim, dims, + (void **)&((*image_info)->tiff_ptr)); +} \ No newline at end of file diff --git a/tools/llsm/parallelReadTiff.h b/tools/llsm/parallelReadTiff.h new file mode 100644 index 000000000..081640110 --- /dev/null +++ b/tools/llsm/parallelReadTiff.h @@ -0,0 +1,34 @@ +#ifndef PARALLELREADTIFF_H +#define PARALLELREADTIFF_H + +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + uint64_t *range; + size_t length; +} parallel_tiff_range_t; + +typedef struct { + uint64_t x; + uint64_t y; + uint64_t z; + uint64_t bits; + uint64_t startSlice; + uint64_t stripeSize; + uint64_t is_imageJ; + uint64_t imageJ_Z; + void * tiff_ptr; + size_t tiff_size; +} image_info_t; + +void parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, + image_info_t **image_info_t); + +#endif // PARALLELREADTIFF_H \ No newline at end of file diff --git a/tools/llsm/pdc_list.c b/tools/llsm/pdc_list.c new file mode 100644 index 000000000..95c6e59e7 --- /dev/null +++ b/tools/llsm/pdc_list.c @@ -0,0 +1,165 @@ +#include "pdc_list.h" + +PDC_LIST * +pdc_list_new() +{ + return pdc_list_create(100, 2.0); +} + +PDC_LIST * +pdc_list_create(size_t initial_capacity, double expansion_factor) +{ + // Allocate memory for the list struct. + PDC_LIST *list = (PDC_LIST *)malloc(sizeof(PDC_LIST)); + if (list == NULL) { + return NULL; + } + + // Allocate memory for the array of items. + list->items = (void **)malloc(initial_capacity * sizeof(void *)); + if (list->items == NULL) { + free(list); + return NULL; + } + + // Initialize the list fields. + list->item_count = 0; + list->capacity = initial_capacity; + list->expansion_factor = expansion_factor; + + return list; +} + +void +pdc_list_destroy(PDC_LIST *list) +{ + if (list == NULL) { + return; + } + + // Free all allocated memory for each item. + for (size_t i = 0; i < list->item_count; i++) { + free(list->items[i]); + } + + // Free the array of items and the list struct. + free(list->items); + free(list); +} + +void +pdc_list_add(PDC_LIST *list, void *item) +{ + if (list == NULL || item == NULL) { + return; + } + + // Expand the array of items if necessary. + if (list->item_count >= list->capacity) { + list->capacity *= list->expansion_factor; + list->items = (void **)realloc(list->items, list->capacity * sizeof(void *)); + if (list->items == NULL) { + return; + } + } + + // Add the new item to the end of the array. + list->items[list->item_count++] = item; +} + +void * +pdc_list_get(PDC_LIST *list, size_t index) +{ + if (list == NULL || index >= list->item_count) { + return NULL; + } + + // Return a pointer to the item at the given index. + return list->items[index]; +} + +size_t +pdc_list_size(PDC_LIST *list) +{ + if (list == NULL) { + return 0; + } + + // Return the number of items in the list. + return list->item_count; +} + +void +pdc_list_set_expansion_factor(PDC_LIST *list, double expansion_factor) +{ + if (list == NULL) { + return; + } + + // Set the new expansion factor for the list. + list->expansion_factor = expansion_factor; +} + +double +pdc_list_get_expansion_factor(PDC_LIST *list) +{ + if (list == NULL) { + return 0; + } + + // Return the current expansion factor for the list. + return list->expansion_factor; +} + +PDC_LIST_ITERATOR * +pdc_list_iterator_new(PDC_LIST *list) +{ + if (list == NULL) { + return NULL; + } + + // Allocate memory for the iterator struct. + PDC_LIST_ITERATOR *iterator = (PDC_LIST_ITERATOR *)malloc(sizeof(PDC_LIST_ITERATOR)); + if (iterator == NULL) { + return NULL; + } + + // Initialize the iterator fields. + iterator->list = list; + iterator->index = 0; + + return iterator; +} + +void +pdc_list_iterator_destroy(PDC_LIST_ITERATOR *iterator) +{ + if (iterator == NULL) { + return; + } + + // Free the iterator struct. + free(iterator); +} + +void * +pdc_list_iterator_next(PDC_LIST_ITERATOR *iterator) +{ + if (iterator == NULL) { + return NULL; + } + + // Return the next item in the list. + return pdc_list_get(iterator->list, iterator->index++); +} + +int +pdc_list_iterator_has_next(PDC_LIST_ITERATOR *iterator) +{ + if (iterator == NULL) { + return 0; + } + + // Return true if there are more items in the list. + return iterator->index < pdc_list_size(iterator->list); +} \ No newline at end of file diff --git a/tools/llsm/pdc_list.h b/tools/llsm/pdc_list.h new file mode 100644 index 000000000..aa71e6124 --- /dev/null +++ b/tools/llsm/pdc_list.h @@ -0,0 +1,110 @@ +#ifndef PDC_LIST_H +#define PDC_LIST_H + +#include + +/** + * A generic list data structure that stores a variable number of items of any type. + */ +typedef struct { + void **items; // Pointer to the array of items. + size_t item_count; // Number of items in the list. + size_t capacity; // Capacity of the array of items. + double expansion_factor; // Factor by which the capacity is expanded. +} PDC_LIST; + +/** + * A generic iterator for iterating over the items in a PDC_LIST. + */ +typedef struct { + PDC_LIST *list; // The list being iterated over. + size_t index; // The index of the next item to be returned. +} PDC_LIST_ITERATOR; + +/** + * Creates a new PDC_LIST with default initial capacity 100 and default expansion factor 2.0. + * @return A pointer to the new PDC_LIST. + */ +PDC_LIST *pdc_list_new(); + +/** + * Creates a new PDC_LIST with the given initial capacity and expansion factor. + * @param initial_capacity The initial capacity of the list. + * @param expansion_factor The factor by which the capacity is expanded when the list is full. + * + * @return A pointer to the new PDC_LIST. + */ +PDC_LIST *pdc_list_create(size_t initial_capacity, double expansion_factor); + +/** + * Destroys the given PDC_LIST and frees all allocated memory. + * @param list The PDC_LIST to destroy. + */ +void pdc_list_destroy(PDC_LIST *list); + +/** + * Adds the given item to the end of the given PDC_LIST. + * @param list The PDC_LIST to add the item to. + * @param item The item to add to the PDC_LIST. + * + */ +void pdc_list_add(PDC_LIST *list, void *item); + +/** + * Gets the item at the given index in the given PDC_LIST. + * @param list The PDC_LIST to get the item from. + * @param index The index of the item to get. + * + * @return A pointer to the item at the given index. + */ +void *pdc_list_get(PDC_LIST *list, size_t index); + +/** + * Sets the item at the given index in the given PDC_LIST. + * @param list The PDC_LIST to set the item in. + * + * @return The number of items in the list. + */ +size_t pdc_list_size(PDC_LIST *list); + +/** + * Sets the expansion factor for the given PDC_LIST. + * @param list The PDC_LIST to set the expansion factor for. + * @param expansion_factor The factor by which the capacity is expanded when the list is full. + */ +void pdc_list_set_expansion_factor(PDC_LIST *list, double expansion_factor); + +/** + * Gets the expansion factor for the given PDC_LIST. + * @param list The PDC_LIST to get the expansion factor for. + */ +double pdc_list_get_expansion_factor(PDC_LIST *list); + +/** + * Creates a new PDC_LIST_ITERATOR for the given PDC_LIST. + * @param list The PDC_LIST to create the iterator for. + * @return A pointer to the new PDC_LIST_ITERATOR. + */ +PDC_LIST_ITERATOR *pdc_list_iterator_new(PDC_LIST *list); + +/** + * Destroys the given PDC_LIST_ITERATOR and frees all allocated memory. + * @param iterator The PDC_LIST_ITERATOR to destroy. + */ +void pdc_list_iterator_destroy(PDC_LIST_ITERATOR *iterator); + +/** + * Returns the next item in the PDC_LIST_ITERATOR. + * @param iterator The PDC_LIST_ITERATOR to get the next item from. + * @return A pointer to the next item in the PDC_LIST_ITERATOR. + */ +void *pdc_list_iterator_next(PDC_LIST_ITERATOR *iterator); + +/** + * Returns true if the PDC_LIST_ITERATOR has more items. + * @param iterator The PDC_LIST_ITERATOR to check. + * @return True if the PDC_LIST_ITERATOR has more items. + */ +int pdc_list_iterator_has_next(PDC_LIST_ITERATOR *iterator); + +#endif // PDC_LIST_H \ No newline at end of file diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c new file mode 100644 index 000000000..d777812c9 --- /dev/null +++ b/tools/llsm_importer.c @@ -0,0 +1,374 @@ +#include +#include +#include +#include +#include + +#ifndef ENABLE_MPI +#define ENABLE_MPI +#endif + +#ifdef ENABLE_MPI +#include "mpi.h" +// #undef ENABLE_MPI +#endif + +#include "pdc.h" +// #include "pdc_client_server_common.h" +// #include "pdc_client_connect.h" + +#include "llsm/parallelReadTiff.h" +#include "llsm/pdc_list.h" +#include "llsm/csvReader.h" +#include + +typedef struct llsm_importer_args_t { + char * directory_path; + csv_header_t *csv_header; +} llsm_importer_args_t; + +int rank = 0, size = 1; + +pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; + +int +parse_console_args(int argc, char *argv[], char **file_name) +{ + int c, parse_code = -1; + + while ((c = getopt(argc, argv, "f:")) != -1) { + switch (c) { + case 'f': + *file_name = optarg; + parse_code = 0; + break; + default: + fprintf(stderr, "Usage: %s [-f filename]\n", argv[0]); + parse_code = -1; + exit(EXIT_FAILURE); + } + } + return parse_code; +} + +void +import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) +{ + struct timespec start, end; + double duration; + + clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation + + obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); + + psize_t ndims = 3; + uint64_t offsets[3] = {0, 0, 0}; + // FIXME: we should support uint64_t. + uint64_t dims[3] = {image_info->x, image_info->y, image_info->z}; + + // psize_t ndims = 1; + // uint64_t offsets[1] = {0}; + // // FIXME: we should support uint64_t. + // uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; + + // FIXME: we should change the ndims parameter to psize_t type. + PDCprop_set_obj_dims(obj_prop_g, (PDC_int_t)ndims, dims); + pdc_var_type_t pdc_type = PDC_UNKNOWN; + switch (image_info->bits) { + case 8: + pdc_type = PDC_UINT8; + break; + case 16: + pdc_type = PDC_UINT16; + break; + case 32: + pdc_type = PDC_FLOAT; + break; + case 64: + pdc_type = PDC_DOUBLE; + break; + default: + printf("Error: unsupported data type.\n"); + exit(-1); + } + PDCprop_set_obj_type(obj_prop_g, pdc_type); + PDCprop_set_obj_time_step(obj_prop_g, 0); + PDCprop_set_obj_user_id(obj_prop_g, getuid()); + PDCprop_set_obj_app_name(obj_prop_g, "LLSM"); + + // uint64_t *offsets = (uint64_t *)malloc(sizeof(uint64_t) * ndims); + // uint64_t *num_bytes = (uint64_t *)malloc(sizeof(uint64_t) * ndims); + // for (int i = 0; i < ndims; i++) { + // offsets[i] = 0; + // num_bytes[i] = dims[i] * image_info->bits / 8; + // } + + // create object + // FIXME: There are many attributes currently in one file name, + // and we should do some research to see what would be a good object name for each image. + pdcid_t cur_obj_g = PDCobj_create(cont_id_g, fileName_cell->field_value, obj_prop_g); + + // write data to object + pdcid_t local_region = PDCregion_create(ndims, offsets, dims); + pdcid_t remote_region = PDCregion_create(ndims, offsets, dims); + pdcid_t transfer_request = + PDCregion_transfer_create(image_info->tiff_ptr, PDC_WRITE, cur_obj_g, local_region, remote_region); + PDCregion_transfer_start(transfer_request); + PDCregion_transfer_wait(transfer_request); + + clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation + duration = (end.tv_sec - start.tv_sec) * 1e9 + + (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds + + printf("[Rank %4d] Region Transfer for object %s [%d Bytes] Done! Time taken: %.4f seconds\n", rank, + fileName_cell->field_value, image_info->tiff_size, duration / 1e9); + + // add metadata tags based on the csv row + csv_cell_t *cell = fileName_cell; + while (cell != NULL) { + char *field_name = cell->header->field_name; + char data_type = cell->header->field_type; + char *field_value = cell->field_value; + switch (data_type) { + case 'i': + int ivalue = atoi(field_value); + PDCobj_put_tag(cur_obj_g, field_name, &ivalue, sizeof(int)); + break; + case 'f': + double fvalue = atof(field_value); + PDCobj_put_tag(cur_obj_g, field_name, &fvalue, sizeof(double)); + break; + case 's': + PDCobj_put_tag(cur_obj_g, field_name, field_value, sizeof(char) * strlen(field_value)); + break; + default: + break; + } + cell = cell->next; + } + + // add extra metadata tags based on the image_info struct + PDCobj_put_tag(cur_obj_g, "x", &(image_info->x), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "y", &(image_info->y), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "z", &(image_info->z), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "bits", &(image_info->bits), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "startSlice", &(image_info->startSlice), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "stripeSize", &(image_info->stripeSize), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "is_imageJ", &(image_info->is_imageJ), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "imageJ_Z", &(image_info->imageJ_Z), sizeof(uint64_t)); + + // close object + PDCobj_close(cur_obj_g); + + // get timing + clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation + duration = (end.tv_sec - start.tv_sec) * 1e9 + + (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds + + printf("[Rank %4d] Create object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, + duration / 1e9); + + // free memory + // free(offsets); + // free(num_bytes); + // PDCregion_close(local_region); + // PDCregion_close(remote_region); + // PDCregion_transfer_close(transfer_request); + PDCprop_close(obj_prop_g); +} + +void +on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) +{ + csv_print_row(row, 1); + + char *dirname = strdup(llsm_args->directory_path); + char filepath[256]; + // calling tiff loading process. + image_info_t * image_info = NULL; + int i = 0; + struct timespec start, end; + double duration; + // Filepath,Filename,StageX_um_,StageY_um_,StageZ_um_,ObjectiveX_um_,ObjectiveY_um_,ObjectiveZ_um_ + + // get the file name from the csv row + csv_cell_t *fileName_cell = csv_get_field_value_by_name(row, llsm_args->csv_header, "Filename"); + + // check if the path ends with a forward slash + if (dirname[strlen(dirname) - 1] != '/') { + strcat(dirname, "/"); // add a forward slash to the end of the path + } + + strcpy(filepath, dirname); // copy the directory path to the file path + strcat(filepath, fileName_cell->field_value); // concatenate the file name to the file path + + clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation + + parallel_TIFF_load(filepath, 1, NULL, &image_info); + + clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation + + duration = (end.tv_sec - start.tv_sec) * 1e9 + + (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds + + printf("[Rand %4d] Read %s Done! Time taken: %.4f seconds\n", rank, filepath, duration / 1e9); + + if (image_info == NULL || image_info->tiff_ptr == NULL) { + return; + } + + printf("first few bytes "); + for (i = 0; i < 10; i++) { + printf("%d ", ((uint8_t *)image_info->tiff_ptr)[i]); + } + printf("\n"); + + // import the image to PDC + import_to_pdc(image_info, fileName_cell); + + // free the image info + free(image_info->tiff_ptr); + free(image_info); + free(dirname); +} + +void +read_txt(char *txtFileName, PDC_LIST *list, int *max_row_length) +{ + FILE *file = fopen(txtFileName, "r"); + + int row_length = 0; + + if (file == NULL) { + printf("Error: could not open file %s\n", txtFileName); + return; + } + char buffer[1024]; + // Read the lines of the file + while (fgets(buffer, sizeof(buffer), file)) { + pdc_list_add(list, strdup(buffer)); + if (row_length < strlen(buffer)) { + row_length = strlen(buffer); + } + } + + fclose(file); + + // Find the maximum row length + *max_row_length = row_length + 5; +} + +int +main(int argc, char *argv[]) +{ + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + char * file_name = NULL; + PDC_LIST * list = pdc_list_new(); + char * csv_line = NULL; + int num_row_read = 0; + csv_header_t * csv_header = NULL; + csv_row_t * csv_row = NULL; + llsm_importer_args_t *llsm_args = NULL; + int bcast_count = 512; + char csv_field_types[] = {'s', 's', 'f', 'f', 'f', 'f', 'f', 'f'}; + // parse console argument + int parse_code = parse_console_args(argc, argv, &file_name); + if (parse_code) { + return parse_code; + } + char *directory_path = dirname(strdup(file_name)); + + // print file name for validating purpose + printf("Filename: %s\n", file_name ? file_name : "(none)"); + printf("Directory: %s\n", directory_path ? directory_path : "(none)"); + + // create a pdc + pdc_id_g = PDCinit("pdc"); + + // create a container property + cont_prop_g = PDCprop_create(PDC_CONT_CREATE, pdc_id_g); + if (cont_prop_g <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); + + // create a container + cont_id_g = PDCcont_create("c1", cont_prop_g); + if (cont_id_g <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); + + // Rank 0 reads the filename list and distribute data to other ranks + if (rank == 0) { + read_txt(file_name, list, &bcast_count); + // print bcast_count + printf("bcast_count: %d \n", bcast_count); + +#ifdef ENABLE_MPI + // broadcast the number of lines + int num_lines = pdc_list_size(list); + MPI_Bcast(&num_lines, 1, MPI_INT, 0, MPI_COMM_WORLD); + // broadcast the bcast_count + MPI_Bcast(&bcast_count, 1, MPI_INT, 0, MPI_COMM_WORLD); + // broadcast the file names + PDC_LIST_ITERATOR *iter = pdc_list_iterator_new(list); + while (pdc_list_iterator_has_next(iter)) { + char *csv_line = (char *)pdc_list_iterator_next(iter); + MPI_Bcast(csv_line, bcast_count, MPI_CHAR, 0, MPI_COMM_WORLD); + } +#endif + } + else { +#ifdef ENABLE_MPI + // other ranks receive the number of files + int num_lines; + MPI_Bcast(&num_lines, 1, MPI_INT, 0, MPI_COMM_WORLD); + // receive the bcast_count + MPI_Bcast(&bcast_count, 1, MPI_INT, 0, MPI_COMM_WORLD); + // receive the file names + int i; + for (i = 0; i < num_lines; i++) { + csv_line = (char *)malloc(bcast_count * sizeof(char)); + MPI_Bcast(csv_line, bcast_count, MPI_CHAR, 0, MPI_COMM_WORLD); + pdc_list_add(list, csv_line); + } +#endif + } + // parse the csv + csv_table_t *csv_table = csv_parse_list(list, csv_field_types); + if (csv_table == NULL) { + printf("Fail to parse csv file @ line %d!\n", __LINE__); + return -1; + } + llsm_args = (llsm_importer_args_t *)malloc(sizeof(llsm_importer_args_t)); + llsm_args->directory_path = directory_path; + llsm_args->csv_header = csv_table->first_header; + + // go through the csv table + csv_row_t *current_row = csv_table->first_row; + while (current_row != NULL) { + if (num_row_read % size == rank) { + on_csv_row(current_row, llsm_args); + } + num_row_read++; + current_row = current_row->next; + } + + csv_free_table(csv_table); + + // close the container + PDCcont_close(cont_id_g); + // close the container property + PDCprop_close(cont_prop_g); + // close the pdc + PDCclose(pdc_id_g); + +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return 0; +} diff --git a/tools/pdc_export.c b/tools/pdc_export.c index e46c17f2a..a67f444e3 100644 --- a/tools/pdc_export.c +++ b/tools/pdc_export.c @@ -8,16 +8,16 @@ #include #include "hdf5.h" -#define ENABLE_MPI 1 +// #define ENABLE_MPI 1 #ifdef ENABLE_MPI #include "mpi.h" #endif #include "pdc.h" -#include "pdc_client_server_common.h" -#include "pdc_client_connect.h" -#include "../src/server/include/pdc_server_metadata.h" +// #include "pdc_client_server_common.h" +// #include "pdc_client_connect.h" +// #include "../src/server/include/pdc_server_metadata.h" #include "cjson/cJSON.h" const char *avail_args[] = {"-f"}; diff --git a/tools/pdc_import.c b/tools/pdc_import.c index 9e12d3a84..f51f587c8 100644 --- a/tools/pdc_import.c +++ b/tools/pdc_import.c @@ -1,8 +1,10 @@ #include #include #include +#include +#include -#define ENABLE_MPI 1 +// #define ENABLE_MPI 1 #ifdef ENABLE_MPI #include "mpi.h" @@ -10,13 +12,14 @@ #include "hdf5.h" #include "pdc.h" -#include "pdc_client_server_common.h" -#include "pdc_client_connect.h" +// #include "pdc_client_server_common.h" +// #include "pdc_client_connect.h" #define MAX_NAME 1024 #define MAX_FILES 2500 #define MAX_FILENAME_LEN 64 #define MAX_TAG_SIZE 8192 +#define TAG_LEN_MAX 2048 typedef struct ArrayList { int length; @@ -80,8 +83,7 @@ int ndset_g = 0; /* FILE *summary_fp_g; */ int max_tag_size_g = 0; pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; -struct timeval write_timer_start_g; -struct timeval write_timer_end_g; +struct timespec write_timer_start_g, write_timer_end_g; struct ArrayList *container_names; int overwrite = 0; @@ -246,9 +248,8 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); #endif - struct timeval pdc_timer_start; - struct timeval pdc_timer_end; - gettimeofday(&pdc_timer_start, 0); + struct timespec pdc_timer_start, pdc_timer_end; + clock_gettime(CLOCK_MONOTONIC, &pdc_timer_start); for (i = 0; i < my_count; i++) { filename = my_filenames[i]; @@ -272,7 +273,8 @@ main(int argc, char **argv) #endif // Checkpoint all metadata after import each hdf5 file if (rank == 0) { - PDC_Client_all_server_checkpoint(); + // FIXME: this should be replaced by a function in public headers. + // PDC_Client_all_server_checkpoint(); } /* printf("%s, %d\n", filename, max_tag_size_g); */ /* printf("\n\n======================\nNumber of datasets: %d\n", ndset_g); */ @@ -285,8 +287,10 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); #endif - gettimeofday(&pdc_timer_end, 0); - double write_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); + clock_gettime(CLOCK_MONOTONIC, &pdc_timer_end); + double write_time = + (pdc_timer_end.tv_sec - pdc_timer_start.tv_sec) * 1e9 + + (pdc_timer_end.tv_nsec - pdc_timer_start.tv_nsec); // calculate duration in nanoseconds #ifdef ENABLE_MPI MPI_Reduce(&ndset_g, &total_dset, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); @@ -294,7 +298,8 @@ main(int argc, char **argv) total_dset = ndset_g; #endif if (rank == 0) { - printf("Import %d datasets with %d ranks took %.2f seconds.\n", total_dset, size, write_time); + printf("Import %d datasets with %d ranks took %.2f seconds.\n", total_dset, size, + write_time / 1e9); } } @@ -539,8 +544,8 @@ do_dset(hid_t did, char *name, char *app_name) scan_attrs(did, obj_id); - pdc_metadata_t *meta = NULL; - obj_region.ndim = ndim; + // pdc_metadata_t *meta = NULL; + obj_region.ndim = ndim; for (i = 0; i < ndim; i++) { offset[i] = 0; size[i] = dims[i]; @@ -550,7 +555,7 @@ do_dset(hid_t did, char *name, char *app_name) obj_region.size = size; if (ndset_g == 1) - gettimeofday(&write_timer_start_g, 0); + clock_gettime(CLOCK_MONOTONIC, &write_timer_start_g); /* PDC_Client_query_metadata_name_timestep(dset_name_g, 0, &meta); */ /* if (meta == NULL) */ @@ -566,12 +571,14 @@ do_dset(hid_t did, char *name, char *app_name) // PDC_Client_write_id(obj_id, &obj_region, buf); if (ndset_g % 100 == 0) { - gettimeofday(&write_timer_end_g, 0); - double elapsed_time = PDC_get_elapsed_time_double(&write_timer_start_g, &write_timer_end_g); - printf("Importer%2d: Finished written 100 objects, took %.2f, my total %d\n", rank, elapsed_time, - ndset_g); + clock_gettime(CLOCK_MONOTONIC, &write_timer_end_g); + double elapsed_time = + (write_timer_end_g.tv_sec - write_timer_start_g.tv_sec) * 1e9 + + (write_timer_end_g.tv_nsec - write_timer_start_g.tv_nsec); // calculate duration in nanoseconds; + printf("Importer%2d: Finished written 100 objects, took %.2f, my total %d\n", rank, + elapsed_time / 1e9, ndset_g); fflush(stdout); - gettimeofday(&write_timer_start_g, 0); + clock_gettime(CLOCK_MONOTONIC, &write_timer_start_g); } free(buf); @@ -698,12 +705,15 @@ scan_attrs(hid_t oid, pdcid_t obj_id) void do_attr(hid_t aid, pdcid_t obj_id) { - ssize_t len; - hid_t atype; - hid_t aspace; - char buf[MAX_NAME] = {0}; - char read_buf[TAG_LEN_MAX] = {0}; - pdc_kvtag_t kvtag1; + ssize_t len; + hid_t atype; + hid_t aspace; + char buf[MAX_NAME] = {0}; + char read_buf[TAG_LEN_MAX] = {0}; + // pdc_kvtag_t kvtag1; + char * tag_name; + void * tag_value; + size_t tag_size; /* * Get the name of the attribute. @@ -717,15 +727,15 @@ do_attr(hid_t aid, pdcid_t obj_id) atype = H5Aget_type(aid); H5Aread(aid, atype, read_buf); - kvtag1.name = buf; - kvtag1.value = (void *)read_buf; + tag_name = buf; + tag_value = (void *)read_buf; if (atype == H5T_STRING) { - kvtag1.size = strlen(read_buf) + 1; + tag_size = strlen(read_buf) + 1; } else { - kvtag1.size = H5Tget_size(atype); + tag_size = H5Tget_size(atype); } - PDCobj_put_tag(obj_id, kvtag1.name, kvtag1.value, kvtag1.size); + PDCobj_put_tag(obj_id, tag_name, tag_value, tag_size); /* * Get attribute information: dataspace, data type From 1029c138ef5cfd2e96c23d58949c4b121b5e5102 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 9 May 2023 23:09:18 -0400 Subject: [PATCH 176/216] added a tutorial for llsm_importer --- tools/LLSM_IMPORTER.md | 74 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tools/LLSM_IMPORTER.md diff --git a/tools/LLSM_IMPORTER.md b/tools/LLSM_IMPORTER.md new file mode 100644 index 000000000..d2f6221ec --- /dev/null +++ b/tools/LLSM_IMPORTER.md @@ -0,0 +1,74 @@ +# LLSM_Importer Tutorial + +This is a tutorial for you to run llsm_importer on Perlmutter supercomputer at NERSC. + +## Prerequisite + +Before building and installing LLSM_importer tool, you need to make sure you install PDC correctly. Check out the latest update on the `develop` branch of `PDC`. Please refer to [Proactive Data Containers (PDC) Installation Guide](../README.md) + +Once you finish all the steps in the installation guide above, you should have environment variable `$WORK_SPACE` defined. + +## Installation + +To build and install LLSM_importer, you need to download libtiff 4.4.0 first. + +```bash +cd $WORK_SPACE/source +wget https://download.osgeo.org/libtiff/tiff-4.4.0.tar.gz +tar zxvf tiff-4.4.0.tar.gz +cd tiff-4.4.0 +./configure --prefix=$WORK_SPACE/install/tiff-4.4.0 +make -j 32 install +``` + +Now you should have libtiff 4.4.0 installed and you need to include the path to the library to your environment variables: + +```bash +echo "export TIFF_DIR=$WORK_SPACE/install/tiff-4.4.0" +echo 'export LD_LIBRARY_PATH=$TIFF_DIR/lib:$LD_LIBRARY_PATH' +echo 'export PATH=$TIFF_DIR/include:$TIFF_DIR/lib:$PATH' + +echo "export TIFF_DIR=$WORK_SPACE/install/tiff-4.4.0" >> $WORK_SPACE/pdc_env.sh +echo 'export LD_LIBRARY_PATH=$TIFF_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh +echo 'export PATH=$TIFF_DIR/include:$TIFF_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh +``` + +Copy the 3 export commands on your screen and run them, and next time if you need to rebuild the llsm_importer program, you can run `$WORK_SPACE/pdc_env.sh` again in advance! + +Now, time to build llsm_importer program. + +```bash +mkdir -p $WORK_SPACE/source/pdc/tools/build +cd $WORK_SPACE/source/pdc/tools/build + +cmake ../ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DPDC_DIR=$PDC_DIR -DUSE_LIB_TIFF=ON -DUSE_SYSTEM_HDF5=ON -DUSE_SYSTEM_OPENMP=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR/tools/ -DCMAKE_C_COMPILER=cc + +make -j 32 +``` + +After this, you should be able to see `llsm_importer` artifact under your `$WORK_SPACE/source/pdc/tools/build` directory. + +## Running LLSM_importer + +First, locate the llsm_importer script + +```bash +cd $WORK_SPACE/source/pdc/scripts/llsm_importer +``` + +Modify the template script `template.sh`. + +Change `EXECPATH` to where your `pdc_server.exe` is installed +Change `TOOLPATH` to where your `llsm_importer` artifact is. + +Change `LLSM_DATA_PATH` to where your sample dataset is. For exmaple, + +```bash +LLSM_DATA_PATH=/pscratch/sd/w/wzhang5/data/llsm/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1 +``` + +Note: you may download the sample dataset from the this [link](https://drive.google.com/file/d/19hH7v58iF_QBJ985ajwLD86MMseBH-YR/view?usp=sharing). It is provided with the courtesy of [Advanced BioImaging Center at UC Berkeley](https://mcb.berkeley.edu/faculty/cdb/upadhyayulas). + +Now, run `gen_script.sh` to generate scripts for different settings with various number of servers. + +After this, enter into any directory named with a number, and submit the job with `sbatch` command. From a82d52a175503a4bc137be070c29775e77b96de2 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 9 May 2023 23:11:46 -0400 Subject: [PATCH 177/216] added a tutorial for llsm_importer --- tools/LLSM_IMPORTER.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/LLSM_IMPORTER.md b/tools/LLSM_IMPORTER.md index d2f6221ec..42ded7682 100644 --- a/tools/LLSM_IMPORTER.md +++ b/tools/LLSM_IMPORTER.md @@ -72,3 +72,5 @@ Note: you may download the sample dataset from the this [link](https://drive.goo Now, run `gen_script.sh` to generate scripts for different settings with various number of servers. After this, enter into any directory named with a number, and submit the job with `sbatch` command. + +Note: This program is still under development and changes will be made available from time to time. Please always use the develop branch for a stable version of this llsm_importer tool. \ No newline at end of file From 97eeafeb8a7aad719998fe60eb54219e49b9bc53 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Wed, 10 May 2023 11:38:37 -0400 Subject: [PATCH 178/216] Adding tutorial for llsm_importer tool. (#84) * remove unnecessary install block from CMakeLists.txt * update output * Revert "update output" This reverts commit fe1f8b44995bc0dabd3b957e1032c2da26f56fdd. * build kvtag_add_get_scale * comment off free * update code * 1. kvtag_scale_add_get added \n 2. uint64_t support for obj/tag/query count \n 3. moving work assigning block downwards right before creating objects \n 4. everything is tested working * do while loop added, tested with 1m object and works * 1m objects test works, 10m object test fail as the original also fails * add new executable to test set * enlarge PDC_SERVER_ID_INTERVAL * update code * update console args * add p search test * add console arg for changing number of attributes per object * free allocated memory * fix query count issue * fix attr length definition * code refactored * code refactored * code refactored * code refactored * code refactored * code refactored * fix data type * fix data type * fix data type * add client side statistics * add client side statistics * fix format * clang formatter * update CMake * update CMake * update CMake * free allocated memory properly * clang format * clang format * clang-format-10 * change file name * address review comments * update llsm importer * update llsm importer * update server checkpoint intervals * update gitignore * adding job scripts * adding one debugging msg * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update output for uint64_t * add scripts * update output for uint64_t * update output for uint64_t * update output for uint64_t * update scripts * update scripts * delete debugging message * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * update tag names * update tag names * update query startingpos * update query startingpos * update job scripts * add progressive timing for kvtag_add_get_scale * fix iteration count in final report * update job scripts and benckmark program * update message format * update message format * update message format * update message format * clang format * update job scripts * comment off object/container close procedure in benchmark to save node hours * change the max number of object to 1M * change the max length of attribute value * change the max length of attribute value * llsm tiff import test * llsm tiff import test * llsm tiff import test * llsm tiff import test * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update cmake and llsm_importer * update cmake and llsm_importer * close if in cmake * cmake fix tiff * cmake policy to suppress warning * add pdc include dir * update code * update code * update code * update code * update code * update code * update array generating method * update array generating method * update array generating method * update array generating method * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * fix return type * fix return type * add timing * add timing * fix output * llsm tiff importer 1st version: read csv and import tiff files to PDC, adding metadata available in CSV files and TIFF loader * fix vairable name * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * add scripts * add scripts * add scripts * debugging for nonMPI program * debugging for nonMPI program * debugging for nonMPI program * clang format, without PDC, everything works perfectly. program fails at PDC init stage where PDCprop_create(PDC_CONT_CREATE, pdc) is being created * enable MPI * enable MPI * enlarge BCase size * enlarge BCase size * enlarge BCase size * resolve bcast count * llsm data path in script * llsm data path in script * update csv reader * update csv reader * update csv reader * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * enlarge max write * update pdc * update pdc * update pdc * update pdc * update pdc_import.c * update pdc_import.c * update pdc_export.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update tools/cmake * clang format * clang format * added a tutorial for llsm_importer * added a tutorial for llsm_importer --------- Co-authored-by: Houjun Tang --- tools/LLSM_IMPORTER.md | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tools/LLSM_IMPORTER.md diff --git a/tools/LLSM_IMPORTER.md b/tools/LLSM_IMPORTER.md new file mode 100644 index 000000000..42ded7682 --- /dev/null +++ b/tools/LLSM_IMPORTER.md @@ -0,0 +1,76 @@ +# LLSM_Importer Tutorial + +This is a tutorial for you to run llsm_importer on Perlmutter supercomputer at NERSC. + +## Prerequisite + +Before building and installing LLSM_importer tool, you need to make sure you install PDC correctly. Check out the latest update on the `develop` branch of `PDC`. Please refer to [Proactive Data Containers (PDC) Installation Guide](../README.md) + +Once you finish all the steps in the installation guide above, you should have environment variable `$WORK_SPACE` defined. + +## Installation + +To build and install LLSM_importer, you need to download libtiff 4.4.0 first. + +```bash +cd $WORK_SPACE/source +wget https://download.osgeo.org/libtiff/tiff-4.4.0.tar.gz +tar zxvf tiff-4.4.0.tar.gz +cd tiff-4.4.0 +./configure --prefix=$WORK_SPACE/install/tiff-4.4.0 +make -j 32 install +``` + +Now you should have libtiff 4.4.0 installed and you need to include the path to the library to your environment variables: + +```bash +echo "export TIFF_DIR=$WORK_SPACE/install/tiff-4.4.0" +echo 'export LD_LIBRARY_PATH=$TIFF_DIR/lib:$LD_LIBRARY_PATH' +echo 'export PATH=$TIFF_DIR/include:$TIFF_DIR/lib:$PATH' + +echo "export TIFF_DIR=$WORK_SPACE/install/tiff-4.4.0" >> $WORK_SPACE/pdc_env.sh +echo 'export LD_LIBRARY_PATH=$TIFF_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh +echo 'export PATH=$TIFF_DIR/include:$TIFF_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh +``` + +Copy the 3 export commands on your screen and run them, and next time if you need to rebuild the llsm_importer program, you can run `$WORK_SPACE/pdc_env.sh` again in advance! + +Now, time to build llsm_importer program. + +```bash +mkdir -p $WORK_SPACE/source/pdc/tools/build +cd $WORK_SPACE/source/pdc/tools/build + +cmake ../ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DPDC_DIR=$PDC_DIR -DUSE_LIB_TIFF=ON -DUSE_SYSTEM_HDF5=ON -DUSE_SYSTEM_OPENMP=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR/tools/ -DCMAKE_C_COMPILER=cc + +make -j 32 +``` + +After this, you should be able to see `llsm_importer` artifact under your `$WORK_SPACE/source/pdc/tools/build` directory. + +## Running LLSM_importer + +First, locate the llsm_importer script + +```bash +cd $WORK_SPACE/source/pdc/scripts/llsm_importer +``` + +Modify the template script `template.sh`. + +Change `EXECPATH` to where your `pdc_server.exe` is installed +Change `TOOLPATH` to where your `llsm_importer` artifact is. + +Change `LLSM_DATA_PATH` to where your sample dataset is. For exmaple, + +```bash +LLSM_DATA_PATH=/pscratch/sd/w/wzhang5/data/llsm/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1 +``` + +Note: you may download the sample dataset from the this [link](https://drive.google.com/file/d/19hH7v58iF_QBJ985ajwLD86MMseBH-YR/view?usp=sharing). It is provided with the courtesy of [Advanced BioImaging Center at UC Berkeley](https://mcb.berkeley.edu/faculty/cdb/upadhyayulas). + +Now, run `gen_script.sh` to generate scripts for different settings with various number of servers. + +After this, enter into any directory named with a number, and submit the job with `sbatch` command. + +Note: This program is still under development and changes will be made available from time to time. Please always use the develop branch for a stable version of this llsm_importer tool. \ No newline at end of file From 152519b284e16425f04d39ea4ba332882ef10205 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 12 May 2023 14:22:03 -0400 Subject: [PATCH 179/216] make sure the line feed is included for string attribute --- tools/llsm_importer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index d777812c9..1fd61dc9b 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -139,7 +139,7 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) PDCobj_put_tag(cur_obj_g, field_name, &fvalue, sizeof(double)); break; case 's': - PDCobj_put_tag(cur_obj_g, field_name, field_value, sizeof(char) * strlen(field_value)); + PDCobj_put_tag(cur_obj_g, field_name, field_value, sizeof(char) * (strlen(field_value) + 1)); break; default: break; From 27621e23877e3603201dce7527950d0ac2daa18b Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 16 May 2023 11:17:15 -0400 Subject: [PATCH 180/216] update timing for overall completion time --- tools/llsm_importer.c | 82 ++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index 1fd61dc9b..a2a00dddd 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -29,7 +29,16 @@ typedef struct llsm_importer_args_t { int rank = 0, size = 1; -pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; +pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; +struct timespec ts; + +double +getDoubleTimestamp() +{ + clock_gettime(CLOCK_MONOTONIC, &ts); + double timestamp = (double)ts.tv_sec + (double)ts.tv_nsec / 1e9; + return timestamp; +} int parse_console_args(int argc, char *argv[], char **file_name) @@ -54,10 +63,9 @@ parse_console_args(int argc, char *argv[], char **file_name) void import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) { - struct timespec start, end; - double duration; + double duration, start; - clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation + start = getDoubleTimestamp(); // start timing the operation obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); @@ -116,12 +124,10 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) PDCregion_transfer_start(transfer_request); PDCregion_transfer_wait(transfer_request); - clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation - duration = (end.tv_sec - start.tv_sec) * 1e9 + - (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds + duration = getDoubleTimestamp() - start; // end timing the operation and calculate duration in nanoseconds - printf("[Rank %4d] Region Transfer for object %s [%d Bytes] Done! Time taken: %.4f seconds\n", rank, - fileName_cell->field_value, image_info->tiff_size, duration / 1e9); + printf("[Rank %4d] Region_Transfer %s_[%d_Bytes] Done! Time taken: %.4f seconds\n", rank, + fileName_cell->field_value, image_info->tiff_size, duration); // add metadata tags based on the csv row csv_cell_t *cell = fileName_cell; @@ -161,12 +167,10 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) PDCobj_close(cur_obj_g); // get timing - clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation - duration = (end.tv_sec - start.tv_sec) * 1e9 + - (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds + duration = getDoubleTimestamp() - start; // end timing the operation calculate duration in nanoseconds - printf("[Rank %4d] Create object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, - duration / 1e9); + printf("[Rank %4d] Create_object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, + duration); // free memory // free(offsets); @@ -185,10 +189,9 @@ on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) char *dirname = strdup(llsm_args->directory_path); char filepath[256]; // calling tiff loading process. - image_info_t * image_info = NULL; - int i = 0; - struct timespec start, end; - double duration; + image_info_t *image_info = NULL; + int i = 0; + double duration, start; // Filepath,Filename,StageX_um_,StageY_um_,StageZ_um_,ObjectiveX_um_,ObjectiveY_um_,ObjectiveZ_um_ // get the file name from the csv row @@ -202,16 +205,13 @@ on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) strcpy(filepath, dirname); // copy the directory path to the file path strcat(filepath, fileName_cell->field_value); // concatenate the file name to the file path - clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation + start = getDoubleTimestamp(); // start timing the operation parallel_TIFF_load(filepath, 1, NULL, &image_info); - clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation + duration = getDoubleTimestamp() - start; // end timing the operation and calculate duration in nanoseconds - duration = (end.tv_sec - start.tv_sec) * 1e9 + - (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds - - printf("[Rand %4d] Read %s Done! Time taken: %.4f seconds\n", rank, filepath, duration / 1e9); + printf("[Rank %4d] Read %s Done! Time taken: %.4f seconds\n", rank, filepath, duration); if (image_info == NULL || image_info->tiff_ptr == NULL) { return; @@ -268,14 +268,15 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &size); #endif - char * file_name = NULL; - PDC_LIST * list = pdc_list_new(); - char * csv_line = NULL; - int num_row_read = 0; - csv_header_t * csv_header = NULL; - csv_row_t * csv_row = NULL; - llsm_importer_args_t *llsm_args = NULL; - int bcast_count = 512; + char * file_name = NULL; + PDC_LIST * list = pdc_list_new(); + char * csv_line = NULL; + int num_row_read = 0; + csv_header_t * csv_header = NULL; + csv_row_t * csv_row = NULL; + llsm_importer_args_t *llsm_args = NULL; + int bcast_count = 512; + double duration = 0, start = 0; char csv_field_types[] = {'s', 's', 'f', 'f', 'f', 'f', 'f', 'f'}; // parse console argument int parse_code = parse_console_args(argc, argv, &file_name); @@ -347,6 +348,12 @@ main(int argc, char *argv[]) llsm_args->directory_path = directory_path; llsm_args->csv_header = csv_table->first_header; +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + start = MPI_Wtime(); +#else + start = getDoubleTimestamp(); +#endif // go through the csv table csv_row_t *current_row = csv_table->first_row; while (current_row != NULL) { @@ -357,6 +364,17 @@ main(int argc, char *argv[]) current_row = current_row->next; } +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + duration = MPI_Wtime() - start; +#else + duration = getDoubleTimestamp() - start; +#endif + + if (rank == 0) { + printf("[Completion Time] LLSM IMPORTER FINISHES! Time taken: %.4f seconds\n", rank, duration); + } + csv_free_table(csv_table); // close the container From 22b98ac1ea34510d9a3d4ec5109e860fded1e607 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Fri, 19 May 2023 12:31:26 -0700 Subject: [PATCH 181/216] Update .gitlab-ci.yml removing Cori Remove Cori CI in advance of Cori's decommission by the end of the month. --- .gitlab-ci.yml | 173 ------------------------------------------------- 1 file changed, 173 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5fde85406..03c29ca4a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,26 +7,6 @@ stages: - test - metrics -cori-build: - stage: build - tags: - - cori - variables: - SCHEDULER_PARAMETERS: "-C haswell --qos=debug -N 1 -t 00:30:00 --gres=craynetwork:3" - LIBFABRIC_DIR: "/global/cfs/cdirs/m1248/pdc/libfabric-1.12.1/install" - MERCURY_DIR: "/global/cfs/cdirs/m1248/pdc/mercury-2.0.0/install" - script: - - module list - - mkdir -p ${PDC_BUILD_PATH}/cori - - cd ${PDC_BUILD_PATH}/cori - - cmake ../.. -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DPDC_ENABLE_MPI=ON -DPDC_ENABLE_TIMING=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=cc -DCMAKE_C_FLAGS=-dynamic -DMPI_RUN_CMD=srun -DPDC_ENABLE_LUSTRE=ON -DPDC_DISABLE_CHECKPOINT=ON -DCMAKE_INSTALL_PREFIX=${PDC_INSTALL_PATH}/cori - - make -j - - make install - artifacts: - paths: - - ${PDC_BUILD_PATH}/cori - - ${PDC_INSTALL_PATH}/cori - perlmutter-build: stage: build tags: @@ -48,119 +28,6 @@ perlmutter-build: - ${PDC_BUILD_PATH}/perlmutter - ${PDC_INSTALL_PATH}/perlmutter -cori-parallel-pdc: - stage: test - needs: - - cori-build - tags: - - cori - variables: - SCHEDULER_PARAMETERS: "-C haswell --qos=debug -N 1 -t 00:30:00 --gres=craynetwork:3" - LIBFABRIC_DIR: "/global/cfs/cdirs/m1248/pdc/libfabric-1.12.1/install" - MERCURY_DIR: "/global/cfs/cdirs/m1248/pdc/mercury-2.0.0/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-paralell-pdc" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-paralell-pdc" - script: - - export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - - cd ${PDC_BUILD_PATH}/cori - - ctest -L parallel_pdc - - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC} - -cori-parallel-obj: - stage: test - needs: - - cori-build - - cori-parallel-pdc - tags: - - cori - variables: - SCHEDULER_PARAMETERS: "-C haswell --qos=debug -N 1 -t 00:30:00 --gres=craynetwork:3" - LIBFABRIC_DIR: "/global/cfs/cdirs/m1248/pdc/libfabric-1.12.1/install" - MERCURY_DIR: "/global/cfs/cdirs/m1248/pdc/mercury-2.0.0/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-paralell-obj" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-paralell-obj" - script: - - export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - - cd ${PDC_BUILD_PATH}/cori - - ctest -L parallel_obj - - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC} - -cori-parallel-cont: - stage: test - needs: - - cori-build - - cori-parallel-pdc - tags: - - cori - variables: - SCHEDULER_PARAMETERS: "-C haswell --qos=debug -N 1 -t 00:30:00 --gres=craynetwork:3" - LIBFABRIC_DIR: "/global/cfs/cdirs/m1248/pdc/libfabric-1.12.1/install" - MERCURY_DIR: "/global/cfs/cdirs/m1248/pdc/mercury-2.0.0/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-paralell-cont" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-paralell-cont" - script: - - export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - - cd ${PDC_BUILD_PATH}/cori - - ctest -L parallel_cont - - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC} - -cori-parallel-prop: - stage: test - needs: - - cori-build - - cori-parallel-pdc - tags: - - cori - variables: - SCHEDULER_PARAMETERS: "-C haswell --qos=debug -N 1 -t 00:30:00 --gres=craynetwork:3" - LIBFABRIC_DIR: "/global/cfs/cdirs/m1248/pdc/libfabric-1.12.1/install" - MERCURY_DIR: "/global/cfs/cdirs/m1248/pdc/mercury-2.0.0/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-paralell-prop" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-paralell-prop" - script: - - export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - - cd ${PDC_BUILD_PATH}/cori - - ctest -L parallel_prop - - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC} - -cori-parallel-region: - stage: test - needs: - - cori-build - - cori-parallel-pdc - tags: - - cori - variables: - SCHEDULER_PARAMETERS: "-C haswell --qos=debug -N 1 -t 00:30:00 --gres=craynetwork:3" - LIBFABRIC_DIR: "/global/cfs/cdirs/m1248/pdc/libfabric-1.12.1/install" - MERCURY_DIR: "/global/cfs/cdirs/m1248/pdc/mercury-2.0.0/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-paralell-region" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-paralell-region" - script: - - export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - - cd ${PDC_BUILD_PATH}/cori - - ctest -L parallel_region_transfer - - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC} - -cori-parallel-region-all: - stage: test - needs: - - cori-build - - cori-parallel-pdc - tags: - - cori - variables: - SCHEDULER_PARAMETERS: "-C haswell --qos=debug -N 1 -t 00:30:00 --gres=craynetwork:3" - LIBFABRIC_DIR: "/global/cfs/cdirs/m1248/pdc/libfabric-1.12.1/install" - MERCURY_DIR: "/global/cfs/cdirs/m1248/pdc/mercury-2.0.0/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-paralell-region-all" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-paralell-region-all" - script: - - export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - - cd ${PDC_BUILD_PATH}/cori - - ctest -L parallel_region_transfer_all - - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC} - perlmutter-parallel-pdc: stage: test needs: @@ -274,46 +141,6 @@ perlmutter-parallel-region-all: - ctest -L parallel_region_transfer_all - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC} -cori-metrics: - stage: metrics - needs: - - cori-build - tags: - - cori - variables: - PDC_N_NODES: 64 - PDC_N_CLIENTS: 31 - SCHEDULER_PARAMETERS: "-C haswell --qos=regular -N ${PDC_N_NODES} -t 00:30:00 --gres=craynetwork:3" - LIBFABRIC_DIR: "/global/cfs/cdirs/m1248/pdc/libfabric-1.12.1/install" - MERCURY_DIR: "/global/cfs/cdirs/m1248/pdc/mercury-2.0.0/install" - PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-metrics" - PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-metrics" - PDC_CLIENT_LOOKUP: "NONE" - PDC_SERVER: "${PDC_BUILD_PATH}/cori/bin/pdc_server.exe" - PDC_SERVER_CLOSE: "${PDC_BUILD_PATH}/cori/bin/close_server" - PDC_CLIENT: "${PDC_BUILD_PATH}/cori/bin/vpicio_mts" - PDC_JOB_OUTPUT: "pdc-metrics.log" - script: - - module load python - - export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH" - - cd ${PDC_BUILD_PATH}/cori - - let TOTAL_PROCESSES=$PDC_N_CLIENTS*$PDC_N_NODES - - echo "Starting PDC servers..." - - export FI_CXI_DEFAULT_VNI=0 - - srun --mem=25600 --cpu_bind=cores --gres=craynetwork:1 --overlap -u -o ${PDC_JOB_OUTPUT} --open-mode=append -N ${PDC_N_NODES} -n ${PDC_N_NODES} -c 1 ${PDC_SERVER} & - - echo "Starting application..." - - export FI_CXI_DEFAULT_VNI=1 - - srun --mem=25600 --cpu_bind=cores --gres=craynetwork:1 --overlap -u -o ${PDC_JOB_OUTPUT} --open-mode=append -N ${PDC_N_NODES} -n ${TOTAL_PROCESSES} -c 1 ${PDC_CLIENT} 8388608 5 20 - - echo "Closing PDC servers..." - - export FI_CXI_DEFAULT_VNI=2 - - srun --mem=25600 --cpu_bind=cores --gres=craynetwork:1 --overlap -u -o ${PDC_JOB_OUTPUT} --open-mode=append -N ${PDC_N_NODES} -n ${PDC_N_NODES} -c 1 ${PDC_SERVER_CLOSE} - - echo "Installing dependencies..." - - pip install pydrive gspread gspread-dataframe google - - echo "Storing PDC metrics..." - - python3 ../../.github/workflows/store-metrics.py Cori ${PDC_JOB_OUTPUT} - - echo "Removing files..." - - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC} - perlmutter-metrics: stage: metrics needs: From 37dd453dec41614407102e3c1a2d75c8feda83c7 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Fri, 19 May 2023 14:30:14 -0700 Subject: [PATCH 182/216] Remove unnecessary fflush call Signed-off-by: Chen Wang --- src/api/pdc_client_connect.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index 9079e04f9..bafaef938 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -244,7 +244,6 @@ PDC_Client_check_response(hg_context_t **hg_context) ret_value = SUCCEED; - fflush(stdout); FUNC_LEAVE(ret_value); } From 108bcc5ce05348368809c5f0e07036827c244ba1 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Tue, 23 May 2023 10:34:23 -0400 Subject: [PATCH 183/216] update formatting --- tools/llsm_importer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index a2a00dddd..ff358fcd3 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -352,7 +352,7 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); start = MPI_Wtime(); #else - start = getDoubleTimestamp(); + start = getDoubleTimestamp(); #endif // go through the csv table csv_row_t *current_row = csv_table->first_row; @@ -374,7 +374,7 @@ main(int argc, char *argv[]) if (rank == 0) { printf("[Completion Time] LLSM IMPORTER FINISHES! Time taken: %.4f seconds\n", rank, duration); } - + // free memory for csv table csv_free_table(csv_table); // close the container From d53a119bc0a6e9419894fa9d1203150fcee62266 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Tue, 23 May 2023 14:34:44 -0700 Subject: [PATCH 184/216] LLSM_importer Tutorial and Timing for job completion time. (#86) * remove unnecessary install block from CMakeLists.txt * update output * Revert "update output" This reverts commit fe1f8b44995bc0dabd3b957e1032c2da26f56fdd. * build kvtag_add_get_scale * comment off free * update code * 1. kvtag_scale_add_get added \n 2. uint64_t support for obj/tag/query count \n 3. moving work assigning block downwards right before creating objects \n 4. everything is tested working * do while loop added, tested with 1m object and works * 1m objects test works, 10m object test fail as the original also fails * add new executable to test set * enlarge PDC_SERVER_ID_INTERVAL * update code * update console args * add p search test * add console arg for changing number of attributes per object * free allocated memory * fix query count issue * fix attr length definition * code refactored * code refactored * code refactored * code refactored * code refactored * code refactored * fix data type * fix data type * fix data type * add client side statistics * add client side statistics * fix format * clang formatter * update CMake * update CMake * update CMake * free allocated memory properly * clang format * clang format * clang-format-10 * change file name * address review comments * update llsm importer * update llsm importer * update server checkpoint intervals * update gitignore * adding job scripts * adding one debugging msg * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update output for uint64_t * add scripts * update output for uint64_t * update output for uint64_t * update output for uint64_t * update scripts * update scripts * delete debugging message * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * update tag names * update tag names * update query startingpos * update query startingpos * update job scripts * add progressive timing for kvtag_add_get_scale * fix iteration count in final report * update job scripts and benckmark program * update message format * update message format * update message format * update message format * clang format * update job scripts * comment off object/container close procedure in benchmark to save node hours * change the max number of object to 1M * change the max length of attribute value * change the max length of attribute value * llsm tiff import test * llsm tiff import test * llsm tiff import test * llsm tiff import test * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update cmake and llsm_importer * update cmake and llsm_importer * close if in cmake * cmake fix tiff * cmake policy to suppress warning * add pdc include dir * update code * update code * update code * update code * update code * update code * update array generating method * update array generating method * update array generating method * update array generating method * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * fix return type * fix return type * add timing * add timing * fix output * llsm tiff importer 1st version: read csv and import tiff files to PDC, adding metadata available in CSV files and TIFF loader * fix vairable name * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * add scripts * add scripts * add scripts * debugging for nonMPI program * debugging for nonMPI program * debugging for nonMPI program * clang format, without PDC, everything works perfectly. program fails at PDC init stage where PDCprop_create(PDC_CONT_CREATE, pdc) is being created * enable MPI * enable MPI * enlarge BCase size * enlarge BCase size * enlarge BCase size * resolve bcast count * llsm data path in script * llsm data path in script * update csv reader * update csv reader * update csv reader * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * enlarge max write * update pdc * update pdc * update pdc * update pdc * update pdc_import.c * update pdc_import.c * update pdc_export.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update tools/cmake * clang format * clang format * added a tutorial for llsm_importer * added a tutorial for llsm_importer * make sure the line feed is included for string attribute * update timing for overall completion time * update formatting --------- Co-authored-by: Houjun Tang --- tools/llsm_importer.c | 84 ++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index d777812c9..ff358fcd3 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -29,7 +29,16 @@ typedef struct llsm_importer_args_t { int rank = 0, size = 1; -pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; +pdcid_t pdc_id_g = 0, cont_prop_g = 0, cont_id_g = 0, obj_prop_g = 0; +struct timespec ts; + +double +getDoubleTimestamp() +{ + clock_gettime(CLOCK_MONOTONIC, &ts); + double timestamp = (double)ts.tv_sec + (double)ts.tv_nsec / 1e9; + return timestamp; +} int parse_console_args(int argc, char *argv[], char **file_name) @@ -54,10 +63,9 @@ parse_console_args(int argc, char *argv[], char **file_name) void import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) { - struct timespec start, end; - double duration; + double duration, start; - clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation + start = getDoubleTimestamp(); // start timing the operation obj_prop_g = PDCprop_create(PDC_OBJ_CREATE, pdc_id_g); @@ -116,12 +124,10 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) PDCregion_transfer_start(transfer_request); PDCregion_transfer_wait(transfer_request); - clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation - duration = (end.tv_sec - start.tv_sec) * 1e9 + - (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds + duration = getDoubleTimestamp() - start; // end timing the operation and calculate duration in nanoseconds - printf("[Rank %4d] Region Transfer for object %s [%d Bytes] Done! Time taken: %.4f seconds\n", rank, - fileName_cell->field_value, image_info->tiff_size, duration / 1e9); + printf("[Rank %4d] Region_Transfer %s_[%d_Bytes] Done! Time taken: %.4f seconds\n", rank, + fileName_cell->field_value, image_info->tiff_size, duration); // add metadata tags based on the csv row csv_cell_t *cell = fileName_cell; @@ -139,7 +145,7 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) PDCobj_put_tag(cur_obj_g, field_name, &fvalue, sizeof(double)); break; case 's': - PDCobj_put_tag(cur_obj_g, field_name, field_value, sizeof(char) * strlen(field_value)); + PDCobj_put_tag(cur_obj_g, field_name, field_value, sizeof(char) * (strlen(field_value) + 1)); break; default: break; @@ -161,12 +167,10 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) PDCobj_close(cur_obj_g); // get timing - clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation - duration = (end.tv_sec - start.tv_sec) * 1e9 + - (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds + duration = getDoubleTimestamp() - start; // end timing the operation calculate duration in nanoseconds - printf("[Rank %4d] Create object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, - duration / 1e9); + printf("[Rank %4d] Create_object %s Done! Time taken: %.4f seconds\n", rank, fileName_cell->field_value, + duration); // free memory // free(offsets); @@ -185,10 +189,9 @@ on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) char *dirname = strdup(llsm_args->directory_path); char filepath[256]; // calling tiff loading process. - image_info_t * image_info = NULL; - int i = 0; - struct timespec start, end; - double duration; + image_info_t *image_info = NULL; + int i = 0; + double duration, start; // Filepath,Filename,StageX_um_,StageY_um_,StageZ_um_,ObjectiveX_um_,ObjectiveY_um_,ObjectiveZ_um_ // get the file name from the csv row @@ -202,16 +205,13 @@ on_csv_row(csv_row_t *row, llsm_importer_args_t *llsm_args) strcpy(filepath, dirname); // copy the directory path to the file path strcat(filepath, fileName_cell->field_value); // concatenate the file name to the file path - clock_gettime(CLOCK_MONOTONIC, &start); // start timing the operation + start = getDoubleTimestamp(); // start timing the operation parallel_TIFF_load(filepath, 1, NULL, &image_info); - clock_gettime(CLOCK_MONOTONIC, &end); // end timing the operation + duration = getDoubleTimestamp() - start; // end timing the operation and calculate duration in nanoseconds - duration = (end.tv_sec - start.tv_sec) * 1e9 + - (end.tv_nsec - start.tv_nsec); // calculate duration in nanoseconds - - printf("[Rand %4d] Read %s Done! Time taken: %.4f seconds\n", rank, filepath, duration / 1e9); + printf("[Rank %4d] Read %s Done! Time taken: %.4f seconds\n", rank, filepath, duration); if (image_info == NULL || image_info->tiff_ptr == NULL) { return; @@ -268,14 +268,15 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &size); #endif - char * file_name = NULL; - PDC_LIST * list = pdc_list_new(); - char * csv_line = NULL; - int num_row_read = 0; - csv_header_t * csv_header = NULL; - csv_row_t * csv_row = NULL; - llsm_importer_args_t *llsm_args = NULL; - int bcast_count = 512; + char * file_name = NULL; + PDC_LIST * list = pdc_list_new(); + char * csv_line = NULL; + int num_row_read = 0; + csv_header_t * csv_header = NULL; + csv_row_t * csv_row = NULL; + llsm_importer_args_t *llsm_args = NULL; + int bcast_count = 512; + double duration = 0, start = 0; char csv_field_types[] = {'s', 's', 'f', 'f', 'f', 'f', 'f', 'f'}; // parse console argument int parse_code = parse_console_args(argc, argv, &file_name); @@ -347,6 +348,12 @@ main(int argc, char *argv[]) llsm_args->directory_path = directory_path; llsm_args->csv_header = csv_table->first_header; +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + start = MPI_Wtime(); +#else + start = getDoubleTimestamp(); +#endif // go through the csv table csv_row_t *current_row = csv_table->first_row; while (current_row != NULL) { @@ -357,6 +364,17 @@ main(int argc, char *argv[]) current_row = current_row->next; } +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + duration = MPI_Wtime() - start; +#else + duration = getDoubleTimestamp() - start; +#endif + + if (rank == 0) { + printf("[Completion Time] LLSM IMPORTER FINISHES! Time taken: %.4f seconds\n", rank, duration); + } + // free memory for csv table csv_free_table(csv_table); // close the container From f255128bca6f794a882203d0423861632e34063f Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Tue, 23 May 2023 15:22:28 -0700 Subject: [PATCH 185/216] Fix Issue #85, server segfault when another client application with different number of ranks connects to it --- src/api/include/pdc_client_connect.h | 3 +- .../pdc_analysis_and_transforms_connect.c | 4 +- src/api/pdc_client_connect.c | 106 +++++++++--------- src/server/include/pdc_client_server_common.h | 7 ++ src/server/pdc_client_server_common.c | 1 + src/server/pdc_server.c | 10 +- 6 files changed, 71 insertions(+), 60 deletions(-) diff --git a/src/api/include/pdc_client_connect.h b/src/api/include/pdc_client_connect.h index 4b0816996..933f2a2e8 100644 --- a/src/api/include/pdc_client_connect.h +++ b/src/api/include/pdc_client_connect.h @@ -54,6 +54,7 @@ struct _pdc_client_lookup_args { uint32_t server_id; uint32_t client_id; int ret; + int is_init; char * ret_string; char * client_addr; @@ -687,7 +688,7 @@ hg_return_t PDC_Client_get_data_from_server_shm_cb(const struct hg_cb_info *call * * \return Non-negative on success/Negative on failure */ -perr_t PDC_Client_lookup_server(int server_id); +perr_t PDC_Client_lookup_server(int server_id, int is_init); /** * ******** diff --git a/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c b/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c index ac957b394..17bc11d5d 100644 --- a/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c +++ b/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c @@ -101,7 +101,7 @@ PDC_Client_send_iter_recv_id(pdcid_t iter_id, pdcid_t *meta_id) while (pdc_server_info_g[server_id].addr_valid != 1) { if (n_retry > 0) break; - if (PDC_Client_lookup_server(server_id) != SUCCEED) + if (PDC_Client_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_lookup_server", pdc_client_mpi_rank_g); n_retry++; @@ -212,7 +212,7 @@ PDC_Client_register_obj_analysis(struct _pdc_region_analysis_ftn_info *thisFtn, while (pdc_server_info_g[server_id].addr_valid != 1) { if (n_retry > 0) break; - if (PDC_Client_lookup_server(server_id) != SUCCEED) + if (PDC_Client_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_lookup_server", pdc_client_mpi_rank_g); n_retry++; diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index bafaef938..bdf967885 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -785,6 +785,7 @@ client_test_connect_lookup_cb(const struct hg_cb_info *callback_info) in.client_id = pdc_client_mpi_rank_g; in.nclient = pdc_client_mpi_size_g; in.client_addr = client_lookup_args->client_addr; + in.is_init = client_lookup_args->is_init; ret_value = HG_Forward(client_test_handle, client_test_connect_rpc_cb, client_lookup_args, &in); if (ret_value != HG_SUCCESS) @@ -796,7 +797,7 @@ client_test_connect_lookup_cb(const struct hg_cb_info *callback_info) } perr_t -PDC_Client_lookup_server(int server_id) +PDC_Client_lookup_server(int server_id, int is_init) { perr_t ret_value = SUCCEED; hg_return_t hg_ret; @@ -817,6 +818,7 @@ PDC_Client_lookup_server(int server_id) lookup_args.client_id = pdc_client_mpi_rank_g; lookup_args.server_id = server_id; lookup_args.client_addr = self_addr; + lookup_args.is_init = is_init; target_addr_string = pdc_server_info_g[lookup_args.server_id].addr_string; if (is_client_debug_g == 1) { @@ -846,7 +848,7 @@ PDC_Client_lookup_server(int server_id) } perr_t -PDC_Client_try_lookup_server(int server_id) +PDC_Client_try_lookup_server(int server_id, int is_init) { perr_t ret_value = SUCCEED; int n_retry = 1; @@ -859,7 +861,7 @@ PDC_Client_try_lookup_server(int server_id) while (pdc_server_info_g[server_id].addr_valid != 1) { if (n_retry > PDC_MAX_TRIAL_NUM) break; - ret_value = PDC_Client_lookup_server(server_id); + ret_value = PDC_Client_lookup_server(server_id, is_init); if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_lookup_server", pdc_client_mpi_rank_g); n_retry++; @@ -1292,7 +1294,7 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po // Each client connect to its node local server only at start time local_server_id = PDC_get_local_server_id(pdc_client_mpi_rank_g, pdc_nclient_per_server_g, pdc_server_num_g); - if (PDC_Client_try_lookup_server(local_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(local_server_id, 1) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR lookup server %d\n", pdc_client_mpi_rank_g, local_server_id); } @@ -1302,7 +1304,7 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po for (local_server_id = 0; local_server_id < pdc_server_num_g; local_server_id++) { if (pdc_client_mpi_size_g > 1000) PDC_msleep(pdc_client_mpi_rank_g % 300); - if (PDC_Client_try_lookup_server(local_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(local_server_id, 1) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR lookup server %d", pdc_client_mpi_rank_g, local_server_id); } @@ -1643,7 +1645,7 @@ PDC_partial_query(int is_list_all, int user_id, const char *app_name, const char } for (server_id = my_server_start; server_id < my_server_end; server_id++) { - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, query_partial_register_id_g, @@ -1727,7 +1729,7 @@ PDC_Client_query_tag(const char *tags, int *n_res, pdc_metadata_t ***out) *n_res = 0; for (server_id = 0; server_id < pdc_server_num_g; server_id++) { - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, query_partial_register_id_g, @@ -1938,7 +1940,7 @@ PDC_Client_add_tag(pdcid_t obj_id, const char *tag) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_add_tag_register_id_g, @@ -2023,7 +2025,7 @@ PDC_Client_update_metadata(pdc_metadata_t *old, pdc_metadata_t *new) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_update_register_id_g, @@ -2112,7 +2114,7 @@ PDC_Client_delete_metadata_by_id(uint64_t obj_id) else debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_delete_by_id_register_id_g, @@ -2169,7 +2171,7 @@ PDC_Client_delete_metadata(char *delete_name, pdcid_t obj_delete_prop) else debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_delete_register_id_g, @@ -2224,7 +2226,7 @@ PDC_Client_query_metadata_name_only(const char *obj_name, pdc_metadata_t **out) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_query_register_id_g, @@ -2281,7 +2283,7 @@ PDC_Client_query_metadata_name_timestep(const char *obj_name, int time_step, pdc // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_query_register_id_g, @@ -2416,7 +2418,7 @@ PDC_Client_create_cont_id(const char *cont_name, pdcid_t cont_create_prop ATTRIB // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = @@ -2491,7 +2493,7 @@ PDC_Client_obj_reset_dims(const char *obj_name, int time_step, int ndim, uint64_ // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, obj_reset_dims_register_id_g, @@ -2621,7 +2623,7 @@ PDC_Client_send_name_recv_id(const char *obj_name, uint64_t cont_id, pdcid_t obj server_id); fflush(stdout); } - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); // We have already filled in the pdc_server_info_g[server_id].addr in previous @@ -2682,7 +2684,7 @@ PDC_Client_close_all_server() if (pdc_client_mpi_size_g >= pdc_server_num_g) { if (pdc_client_mpi_rank_g < pdc_server_num_g) { server_id = pdc_client_mpi_rank_g; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); @@ -2708,7 +2710,7 @@ PDC_Client_close_all_server() if (pdc_client_mpi_rank_g == 0) { for (i = 0; i < (uint32_t)pdc_server_num_g; i++) { server_id = pdc_server_num_g - 1 - i; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); @@ -2776,7 +2778,7 @@ PDC_Client_buf_unmap(pdcid_t remote_obj_id, pdcid_t remote_reg_id, struct pdc_re // Debug statistics for counting number of messages sent to each server. debug_server_id_count[data_server_id]++; - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[data_server_id].addr, buf_unmap_register_id_g, @@ -2857,7 +2859,7 @@ PDC_Client_flush_obj(uint64_t obj_id) FUNC_ENTER(NULL); for (i = 0; i < (uint32_t)pdc_server_num_g; i++) { server_id = pdc_server_num_g - 1 - i; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, flush_obj_register_id_g, @@ -2897,7 +2899,7 @@ PDC_Client_flush_obj_all() FUNC_ENTER(NULL); for (i = 0; i < (uint32_t)pdc_server_num_g; i++) { server_id = pdc_server_num_g - 1 - i; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, flush_obj_all_register_id_g, @@ -2956,7 +2958,7 @@ PDC_Client_transfer_request_all(int n_objs, pdc_access_t access_type, uint32_t d hg_class = HG_Context_get_class(send_context_g); - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); @@ -3042,7 +3044,7 @@ PDC_Client_transfer_request_metadata_query2(char *buf, uint64_t total_buf_size, hg_class = HG_Context_get_class(send_context_g); - if (PDC_Client_try_lookup_server(metadata_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(metadata_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); hg_ret = HG_Create(send_context_g, pdc_server_info_g[metadata_server_id].addr, @@ -3113,7 +3115,7 @@ PDC_Client_transfer_request_metadata_query(char *buf, uint64_t total_buf_size, i hg_class = HG_Context_get_class(send_context_g); - if (PDC_Client_try_lookup_server(metadata_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(metadata_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); hg_ret = HG_Create(send_context_g, pdc_server_info_g[metadata_server_id].addr, @@ -3184,7 +3186,7 @@ PDC_Client_transfer_request_wait_all(int n_objs, pdcid_t *transfer_request_id, u hg_class = HG_Context_get_class(send_context_g); - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); hg_ret = @@ -3296,7 +3298,7 @@ PDC_Client_transfer_request(void *buf, pdcid_t obj_id, uint32_t data_server_id, pack_region_metadata(remote_ndim, remote_offset, remote_size, &(in.remote_region)); - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); @@ -3366,7 +3368,7 @@ PDC_Client_transfer_request_status(pdcid_t transfer_request_id, uint32_t data_se in.transfer_request_id = transfer_request_id; - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); @@ -3418,7 +3420,7 @@ PDC_Client_transfer_request_wait(pdcid_t transfer_request_id, uint32_t data_serv in.transfer_request_id = transfer_request_id; in.access_type = access_type; - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server @ line %d", pdc_client_mpi_rank_g, __LINE__); @@ -3599,7 +3601,7 @@ PDC_Client_buf_map(pdcid_t local_region_id, pdcid_t remote_obj_id, size_t ndim, else PGOTO_ERROR(FAIL, "mapping for array of dimension greater than 4 is not supproted"); - if (PDC_Client_try_lookup_server(data_server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[data_server_id].addr, buf_map_register_id_g, @@ -3682,7 +3684,7 @@ PDC_Client_region_lock(pdcid_t remote_obj_id, struct _pdc_obj_info *object_info, in.data_unit = PDC_get_var_type_size(data_type); PDC_region_info_t_to_transfer_unit(region_info, &(in.region), in.data_unit); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, region_lock_register_id_g, @@ -3793,7 +3795,7 @@ pdc_region_release_with_server_transform(struct _pdc_obj_info * object_info, unit = type_extent; PDC_region_info_t_to_transfer_unit(region_info, &(in.region), unit); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); // Create a bulk handle for the temp buffer used by the transform @@ -3902,7 +3904,7 @@ pdc_region_release_with_server_analysis(struct _pdc_obj_info * object_info, in.output_obj_id = obj_prop->obj_prop_pub->obj_prop_id; in.output_iter = outputIter->meta_id; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, region_analysis_release_register_id_g, @@ -4002,7 +4004,7 @@ pdc_region_release_with_client_transform(struct _pdc_obj_info * object_info, unit = type_extent; PDC_region_info_t_to_transfer_unit(region_info, &(in.region), unit); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); transform_args.data = data_ptrs[0]; @@ -4334,7 +4336,7 @@ PDC_Client_region_release(pdcid_t remote_obj_id, struct _pdc_obj_info *object_in in.data_unit = PDC_get_var_type_size(data_type); PDC_region_info_t_to_transfer_unit(region_info, &(in.region), in.data_unit); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, region_release_register_id_g, @@ -4557,7 +4559,7 @@ PDC_Client_data_server_read_check(int server_id, uint32_t client_id, pdc_metadat read_size *= region->size[i]; } - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, data_server_read_check_register_id_g, @@ -4704,7 +4706,7 @@ PDC_Client_data_server_read(struct pdc_request *request) PDC_metadata_t_to_transfer_t(meta, &in.meta); PDC_region_info_t_to_transfer(region, &in.region); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, data_server_read_register_id_g, @@ -4824,7 +4826,7 @@ PDC_Client_data_server_write_check(struct pdc_request *request, int *status) write_size *= region->size[i]; } - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, data_server_write_check_register_id_g, @@ -4989,7 +4991,7 @@ PDC_Client_data_server_write(struct pdc_request *request) PDC_metadata_t_to_transfer_t(meta, &in.meta); PDC_region_info_t_to_transfer(region, &in.region); - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, data_server_write_register_id_g, @@ -5360,7 +5362,7 @@ PDC_Client_add_del_objects_to_container(int nobj, uint64_t *obj_ids, uint64_t co // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); // Send the bulk handle to the target with RPC @@ -5478,7 +5480,7 @@ PDC_Client_add_tags_to_container(pdcid_t cont_id, char *tags) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); // Send the bulk handle to the target with RPC @@ -5555,7 +5557,7 @@ PDC_Client_query_container_name(const char *cont_name, uint64_t *cont_meta_id) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, container_query_register_id_g, @@ -5732,7 +5734,7 @@ PDC_Client_query_name_read_entire_obj(int nobj, char **obj_names, void ***out_bu // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); // Send the bulk handle to the target with RPC @@ -5913,7 +5915,7 @@ PDC_Client_server_checkpoint(uint32_t server_id) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, server_checkpoint_rpc_register_id_g, @@ -6005,7 +6007,7 @@ PDC_Client_send_client_shm_info(uint32_t server_id, char *shm_addr, uint64_t siz // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = @@ -6120,7 +6122,7 @@ PDC_send_region_storage_meta_shm(uint32_t server_id, int n, region_storage_meta_ // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, @@ -6428,7 +6430,7 @@ PDC_Client_query_multi_storage_info(int nobj, char **obj_names, region_storage_m send_n_request++; debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); @@ -7005,7 +7007,7 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_add_kvtag_register_id_g, @@ -7102,7 +7104,7 @@ PDC_get_kvtag(pdcid_t obj_id, char *tag_name, pdc_kvtag_t **kvtag, int is_cont) server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_get_kvtag_register_id_g, @@ -7154,7 +7156,7 @@ PDCtag_delete(pdcid_t obj_id, char *tag_name) debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_del_kvtag_register_id_g, @@ -7323,7 +7325,7 @@ PDC_Client_query_kvtag_server(uint32_t server_id, const pdc_kvtag_t *kvtag, int *out = NULL; *n_res = 0; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); hg_ret = HG_Create(send_context_g, pdc_server_info_g[server_id].addr, query_kvtag_register_id_g, @@ -7948,7 +7950,7 @@ PDC_send_data_query(pdc_query_t *query, pdc_query_get_op_t get_op, uint64_t *nhi query_xfer->next_server_id = next_server; query_xfer->prev_server_id = prev_server; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, send_data_query_register_id_g, &handle); @@ -8102,7 +8104,7 @@ PDC_Client_get_sel_data(pdcid_t obj_id, pdc_selection_t *sel, void *data) server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); debug_server_id_count[server_id]++; - if (PDC_Client_try_lookup_server(server_id) != SUCCEED) + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); HG_Create(send_context_g, pdc_server_info_g[server_id].addr, get_sel_data_register_id_g, &handle); diff --git a/src/server/include/pdc_client_server_common.h b/src/server/include/pdc_client_server_common.h index 7be76a1b3..a57e2169f 100644 --- a/src/server/include/pdc_client_server_common.h +++ b/src/server/include/pdc_client_server_common.h @@ -282,6 +282,7 @@ typedef struct { typedef struct { int32_t client_id; int32_t nclient; + int32_t is_init; char client_addr[ADDR_MAX]; } client_test_connect_args; @@ -688,6 +689,7 @@ typedef struct { typedef struct { uint32_t client_id; int32_t nclient; + int is_init; hg_string_t client_addr; } client_test_connect_in_t; @@ -2140,6 +2142,11 @@ hg_proc_client_test_connect_in_t(hg_proc_t proc, void *data) // HG_LOG_ERROR("Proc error"); return ret; } + ret = hg_proc_int32_t(proc, &struct_data->is_init); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } ret = hg_proc_hg_string_t(proc, &struct_data->client_addr); if (ret != HG_SUCCESS) { // HG_LOG_ERROR("Proc error"); diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index 14417485b..a0a7845a4 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -1656,6 +1656,7 @@ HG_TEST_RPC_CB(client_test_connect, handle) #endif args->client_id = in.client_id; args->nclient = in.nclient; + args->is_init = in.is_init; sprintf(args->client_addr, "%s", in.client_addr); #ifdef ENABLE_MULTITHREAD hg_thread_mutex_unlock(&pdc_client_info_mutex_g); diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 1ffddb283..ccd6ca95c 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -82,7 +82,6 @@ pdc_client_info_t * pdc_client_info_g = NULL; pdc_remote_server_info_t *pdc_remote_server_info_g = NULL; char * all_addr_strings_1d_g = NULL; char ** all_addr_strings_g = NULL; -int is_all_client_connected_g = 0; int is_hash_table_init_g = 0; int lustre_stripe_size_mb_g = 16; int lustre_total_ost_g = 0; @@ -265,13 +264,14 @@ PDC_Server_get_client_addr(const struct hg_cb_info *callback_info) hg_thread_mutex_lock(&pdc_client_addr_mutex_g); #endif - if (is_all_client_connected_g == 1) { - printf("==PDC_SERVER[%d]: new application run detected, create new client info\n", pdc_server_rank_g); - fflush(stdout); + if (pdc_client_info_g && in->is_init == 1) { + if (is_debug_g && pdc_server_rank_g == 0) { + printf("==PDC_SERVER[%d]: new application run detected, create new client info\n", pdc_server_rank_g); + fflush(stdout); + } PDC_Server_destroy_client_info(pdc_client_info_g); pdc_client_info_g = NULL; - is_all_client_connected_g = 0; } #ifdef ENABLE_MULTITHREAD From 62798a4fa6fd886487434e7c8072fd489cbe70cb Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 23 May 2023 22:22:57 +0000 Subject: [PATCH 186/216] Committing clang-format changes --- src/server/pdc_server.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index ccd6ca95c..a3020628a 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -78,13 +78,13 @@ pdc_task_list_t *pdc_server_agg_task_head_g = NULL; pdc_task_list_t *pdc_server_s2s_task_head_g = NULL; int pdc_server_task_id_g = PDC_SERVER_TASK_INIT_VALUE; -pdc_client_info_t * pdc_client_info_g = NULL; -pdc_remote_server_info_t *pdc_remote_server_info_g = NULL; -char * all_addr_strings_1d_g = NULL; -char ** all_addr_strings_g = NULL; -int is_hash_table_init_g = 0; -int lustre_stripe_size_mb_g = 16; -int lustre_total_ost_g = 0; +pdc_client_info_t * pdc_client_info_g = NULL; +pdc_remote_server_info_t *pdc_remote_server_info_g = NULL; +char * all_addr_strings_1d_g = NULL; +char ** all_addr_strings_g = NULL; +int is_hash_table_init_g = 0; +int lustre_stripe_size_mb_g = 16; +int lustre_total_ost_g = 0; hg_id_t get_remote_metadata_register_id_g; hg_id_t buf_map_server_register_id_g; @@ -266,12 +266,13 @@ PDC_Server_get_client_addr(const struct hg_cb_info *callback_info) if (pdc_client_info_g && in->is_init == 1) { if (is_debug_g && pdc_server_rank_g == 0) { - printf("==PDC_SERVER[%d]: new application run detected, create new client info\n", pdc_server_rank_g); + printf("==PDC_SERVER[%d]: new application run detected, create new client info\n", + pdc_server_rank_g); fflush(stdout); } PDC_Server_destroy_client_info(pdc_client_info_g); - pdc_client_info_g = NULL; + pdc_client_info_g = NULL; } #ifdef ENABLE_MULTITHREAD From cec387e2d2735d348291f45a7fbf694d444f51b8 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 5 Jun 2023 05:51:40 -0700 Subject: [PATCH 187/216] update metrics --- .github/workflows/store-metrics.py | 59 ++++++++++++++++++++++++++++++ .gitlab-ci.yml | 3 +- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/.github/workflows/store-metrics.py b/.github/workflows/store-metrics.py index fed492af0..9d12b6276 100644 --- a/.github/workflows/store-metrics.py +++ b/.github/workflows/store-metrics.py @@ -154,6 +154,9 @@ if len(obj_create_time_node) > 0: observations = { + 'branch': sys.argv[3], + 'JOBID': sys.argv[4], + 'pdc_metadata_servers': pdc_metadata_servers, 'pdc_metadata_clients': pdc_metadata_clients, @@ -245,3 +248,59 @@ 'values': df_values } ) + + lines = [] + + # Record all the steps + for step in range(0, len(obj_create_time_node)): + line = { + 'branch': sys.argv[3], + 'JOBID': sys.argv[4], + + 'pdc_metadata_servers': pdc_metadata_servers, + 'pdc_metadata_clients': pdc_metadata_clients, + + 'date': str(today), + + 'step': step + 1, + + 'obj_create_time_step': obj_create_time_node[step], + 'xfer_create_time_step': xfer_create_time_node[step], + 'xfer_start_time_step': xfer_start_time_node[step], + 'xfer_wait_time_step': xfer_wait_time_node[step], + 'xfer_close_time_step': xfer_close_time_node[step], + 'obj_close_time_step': obj_close_time_node[step], + 'sleep_time_step': sleep_time_node[step] if len(sleep_time_node) < step else 0 + } + + lines.append(line) + + dataset = pd.DataFrame.from_dict(lines) + + print(dataset) + + dataset.applymap(lambda x: x.strip() if isinstance(x, str) else x) + + df_values = dataset.values.tolist() + + if False: + # Submit the header to the spreadsheet + gs.values_append( + '{} - Steps'.format(sys.argv[1]), + { + 'valueInputOption': 'USER_ENTERED' + }, + { + 'values': [dataset.columns.tolist()] + } + ) + + gs.values_append( + '{} - Steps'.format(sys.argv[1]), + { + 'valueInputOption': 'USER_ENTERED' + }, + { + 'values': df_values + } + ) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 03c29ca4a..c588c2429 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -162,6 +162,7 @@ perlmutter-metrics: PDC_JOB_OUTPUT: "pdc-metrics.log" script: - hostname + - echo "JOBID ${SLURM_JOB_ID}" - export NERSC_HOST=`cat /etc/clustername` - module load python - export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" @@ -176,6 +177,6 @@ perlmutter-metrics: - echo "Installing dependencies..." - pip install pydrive gspread gspread-dataframe google - echo "Storing PDC metrics..." - - python3 ../../.github/workflows/store-metrics.py Perlmutter ${PDC_JOB_OUTPUT} + - python3 ../../.github/workflows/store-metrics.py Perlmutter ${PDC_JOB_OUTPUT} ${CI_COMMIT_BRANCH} ${SLURM_JOB_ID} - echo "Removing files..." - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC} From d7c4f11315eb4eed550a0cd834497acaf46e9703 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 13 Jun 2023 09:56:11 -0700 Subject: [PATCH 188/216] Update .gitlab-ci.yml --- .gitlab-ci.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c588c2429..27e4d751c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,8 @@ perlmutter-build: perlmutter-parallel-pdc: stage: test + rules: + - if: '$METRICS == null' needs: - perlmutter-build tags: @@ -48,6 +50,8 @@ perlmutter-parallel-pdc: perlmutter-parallel-obj: stage: test + rules: + - if: '$METRICS == null' needs: - perlmutter-build - perlmutter-parallel-pdc @@ -67,6 +71,8 @@ perlmutter-parallel-obj: perlmutter-parallel-cont: stage: test + rules: + - if: '$METRICS == null' needs: - perlmutter-build - perlmutter-parallel-pdc @@ -86,6 +92,8 @@ perlmutter-parallel-cont: perlmutter-parallel-prop: stage: test + rules: + - if: '$METRICS == null' needs: - perlmutter-build - perlmutter-parallel-pdc @@ -105,6 +113,8 @@ perlmutter-parallel-prop: perlmutter-parallel-region: stage: test + rules: + - if: '$METRICS == null' needs: - perlmutter-build - perlmutter-parallel-pdc @@ -124,6 +134,8 @@ perlmutter-parallel-region: perlmutter-parallel-region-all: stage: test + rules: + - if: '$METRICS == null' needs: - perlmutter-build - perlmutter-parallel-pdc @@ -143,14 +155,16 @@ perlmutter-parallel-region-all: perlmutter-metrics: stage: metrics + rules: + - if: '$METRICS == "true"' needs: - perlmutter-build tags: - perlmutter variables: - PDC_N_NODES: 64 + PDC_N_NODES: 4 PDC_N_CLIENTS: 127 - SCHEDULER_PARAMETERS: "-A m1248 --qos=regular --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00" + SCHEDULER_PARAMETERS: "-A m1248 --qos=debug --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/m1248/pdc-perlmutter/mercury/install" PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-metrics" From a981716c286dcff60ed93163eb89c3ffbca1d0c3 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 13 Jun 2023 10:06:47 -0700 Subject: [PATCH 189/216] update VPIC output timing precision (#88) * update VPIC output timing precision * update timing to make consistent --- src/tests/bdcats_v2.c | 8 +++---- src/tests/client_server.c | 2 +- src/tests/cont_add_del.c | 4 ++-- src/tests/create_obj_scale.c | 4 ++-- src/tests/data_server_read.c | 2 +- src/tests/data_server_read_multi.c | 2 +- src/tests/data_server_read_vpic_multits.c | 2 +- .../data_server_read_vpic_spatial_multits.c | 2 +- src/tests/data_server_write.c | 2 +- src/tests/data_server_write_multi.c | 2 +- src/tests/data_server_write_vpic_multits.c | 2 +- src/tests/delete_obj_scale.c | 2 +- src/tests/kvtag_add_get_benchmark.c | 24 +++++++++---------- src/tests/kvtag_query_scale.c | 2 +- src/tests/list_all.c | 2 +- src/tests/obj_lock.c | 4 ++-- src/tests/obj_map.c | 4 ++-- src/tests/obj_transformation.c | 6 ++--- src/tests/query_data.c | 2 +- src/tests/query_vpic.c | 6 ++--- src/tests/query_vpic_bin_sds1_nopreload.c | 4 ++-- src/tests/query_vpic_bin_sds1_preload.c | 4 ++-- src/tests/query_vpic_bin_sds_nopreload.c | 4 ++-- src/tests/query_vpic_bin_sds_preload.c | 4 ++-- src/tests/query_vpic_exyz_nopreload.c | 4 ++-- src/tests/query_vpic_exyz_preload.c | 4 ++-- src/tests/query_vpic_multi.c | 6 ++--- src/tests/query_vpic_multi_nopreload.c | 6 ++--- src/tests/query_vpic_multi_nopreload1.c | 6 ++--- src/tests/query_vpic_multi_preload.c | 6 ++--- src/tests/read_obj.c | 4 ++-- src/tests/search_obj.c | 6 ++--- src/tests/search_obj_scale.c | 4 ++-- src/tests/stat_obj.c | 4 ++-- src/tests/update_obj.c | 4 ++-- src/tests/vpicio.c | 10 ++++---- src/tests/vpicio_mts.c | 12 +++++----- src/tests/vpicio_v2.c | 6 ++--- src/tests/write_obj_shared.c | 2 +- 39 files changed, 92 insertions(+), 92 deletions(-) diff --git a/src/tests/bdcats_v2.c b/src/tests/bdcats_v2.c index aeea45134..88a78b094 100644 --- a/src/tests/bdcats_v2.c +++ b/src/tests/bdcats_v2.c @@ -226,7 +226,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to map with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to map with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -275,7 +275,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to lock with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to lock with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -324,7 +324,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to relese lock with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to relese lock with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -372,7 +372,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to read data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to read data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/client_server.c b/src/tests/client_server.c index cd1d8810f..3e2411ce5 100644 --- a/src/tests/client_server.c +++ b/src/tests/client_server.c @@ -237,7 +237,7 @@ main(int argc, char **argv) hg_time_get_current(&end_time); elapsed_time = hg_time_subtract(end_time, start_time); elapsed_time_double = hg_time_to_double(elapsed_time); - printf("Total elapsed time for PDC server connection: %.6fs\n", elapsed_time_double); + printf("Total elapsed time for PDC server connection: %.5e s\n", elapsed_time_double); for (i = 0; i < n_server; i++) { printf("\"%s\" obj_id = %d\n", client_lookup_args[i].obj_name, client_lookup_args[i].obj_id); diff --git a/src/tests/cont_add_del.c b/src/tests/cont_add_del.c index 993a41f16..de55ff6e7 100644 --- a/src/tests/cont_add_del.c +++ b/src/tests/cont_add_del.c @@ -198,7 +198,7 @@ main(int argc, char **argv) ht_total_end.tv_usec - ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("%10d created ... %.4f s\n", i * size, ht_total_sec); + printf("%10d created ... %.5e s\n", i * size, ht_total_sec); fflush(stdout); } #ifdef ENABLE_MPI @@ -215,7 +215,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to create %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to create %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/create_obj_scale.c b/src/tests/create_obj_scale.c index 61244a63d..ebd0d373e 100644 --- a/src/tests/create_obj_scale.c +++ b/src/tests/create_obj_scale.c @@ -201,7 +201,7 @@ main(int argc, char **argv) ht_total_end.tv_usec - ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("%10d created ... %.4f s\n", i * size, ht_total_sec); + printf("%10d created ... %.5e s\n", i * size, ht_total_sec); fflush(stdout); } #ifdef ENABLE_MPI @@ -218,7 +218,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to create %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to create %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/data_server_read.c b/src/tests/data_server_read.c index b57855a58..f56ac5f09 100644 --- a/src/tests/data_server_read.c +++ b/src/tests/data_server_read.c @@ -88,7 +88,7 @@ main(int argc, char **argv) ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to read data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to read data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/data_server_read_multi.c b/src/tests/data_server_read_multi.c index 2081547e2..9d6cdafda 100644 --- a/src/tests/data_server_read_multi.c +++ b/src/tests/data_server_read_multi.c @@ -197,7 +197,7 @@ main(int argc, char **argv) if (rank == 0) { printf( - "Total time read %d ts data each %luMB with %d ranks: %.6f, meta %.2f, wait %.2f, sleep %.2f\n", + "Total time read %d ts data each %luMB with %d ranks: %.5e, meta %.2f, wait %.2f, sleep %.2f\n", ntimestep, size_MB, size, total_elapsed / 1000000.0, total_meta_sec, total_wait_sec, sleepseconds * ntimestep); fflush(stdout); diff --git a/src/tests/data_server_read_vpic_multits.c b/src/tests/data_server_read_vpic_multits.c index 6222c1725..f260e8cfb 100644 --- a/src/tests/data_server_read_vpic_multits.c +++ b/src/tests/data_server_read_vpic_multits.c @@ -322,7 +322,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); #endif if (rank == 0) - printf("Timestep %d: query time %.4f, read time %.4f, wait time %.4f, compute time %.4f\n", ts, + printf("Timestep %d: query time %.5e, read time %.5e, wait time %.5e, compute time %.5e\n", ts, query_time, read_time, wait_time, true_sleep_time); } // end of for ts diff --git a/src/tests/data_server_read_vpic_spatial_multits.c b/src/tests/data_server_read_vpic_spatial_multits.c index 6b3c5a5ff..e667736cf 100644 --- a/src/tests/data_server_read_vpic_spatial_multits.c +++ b/src/tests/data_server_read_vpic_spatial_multits.c @@ -326,7 +326,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); #endif if (rank == 0) - printf("Timestep %d: query time %.4f, read time %.4f, wait time %.4f, compute time %.4f\n", ts, + printf("Timestep %d: query time %.5e, read time %.5e, wait time %.5e, compute time %.5e\n", ts, query_time, read_time, wait_time, true_sleep_time); } // end of for ts diff --git a/src/tests/data_server_write.c b/src/tests/data_server_write.c index b9c46527e..174d780fe 100644 --- a/src/tests/data_server_write.c +++ b/src/tests/data_server_write.c @@ -124,7 +124,7 @@ main(int argc, char **argv) ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to write data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to write data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/data_server_write_multi.c b/src/tests/data_server_write_multi.c index 541365069..fd5a02478 100644 --- a/src/tests/data_server_write_multi.c +++ b/src/tests/data_server_write_multi.c @@ -224,7 +224,7 @@ main(int argc, char **argv) if (rank == 0) { printf( - "Total time write %d ts data each %luMB with %d ranks: %.6f, meta %.2f, wait %.2f, sleep %.2f\n", + "Total time write %d ts data each %luMB with %d ranks: %.5e, meta %.2f, wait %.2f, sleep %.2f\n", ntimestep, size_MB, size, total_elapsed / 1000000.0, total_meta_sec, total_wait_sec, sleepseconds * ntimestep); fflush(stdout); diff --git a/src/tests/data_server_write_vpic_multits.c b/src/tests/data_server_write_vpic_multits.c index 6729c759f..6a943bb58 100644 --- a/src/tests/data_server_write_vpic_multits.c +++ b/src/tests/data_server_write_vpic_multits.c @@ -346,7 +346,7 @@ main(int argc, char **argv) } if (rank == 0) - printf("Timestep %d: create time %.6f, query time %.6f, write time %.6f, wait time %.6f\n", ts, + printf("Timestep %d: create time %.5e, query time %.5e, write time %.5e, wait time %.5e\n", ts, create_time, query_time, write_time, wait_time); } diff --git a/src/tests/delete_obj_scale.c b/src/tests/delete_obj_scale.c index ceff24141..e97aa409f 100644 --- a/src/tests/delete_obj_scale.c +++ b/src/tests/delete_obj_scale.c @@ -191,7 +191,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to create %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to create %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 5cda25433..ee5efdcbf 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -399,12 +399,12 @@ main(int argc, char *argv[]) if (my_rank == 0) { printf("Iteration %" PRIu64 " : Objects: %" PRIu64 - " , Time: %.4f sec. Object throughput in this iteration: " - "%.4f .\n", + " , Time: %.5e sec. Object throughput in this iteration: " + "%.5e .\n", k, n_obj_incr, step_elapse, ((double)n_obj_incr) / step_elapse); printf("Overall %" PRIu64 " : Objects: %" PRIu64 - " , Time: %.4f sec. Overall object throughput: " - "%.4f .\n", + " , Time: %.5e sec. Overall object throughput: " + "%.5e .\n", k, total_object_count, total_object_time, ((double)total_object_count) / total_object_time); } @@ -424,10 +424,10 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) { printf("Iteration %" PRIu64 " : Tags: %" PRIu64 - " , Time: %.4f sec. Tag throughput in this iteration: %.4f .\n", + " , Time: %.5e sec. Tag throughput in this iteration: %.5e .\n", k, n_obj_incr * n_attr, step_elapse, (double)(n_obj_incr * n_attr) / step_elapse); printf("Overall %" PRIu64 " : Tags: %" PRIu64 - " , Time: %.4f sec. Overall tag throughput: %.4f .\n", + " , Time: %.5e sec. Overall tag throughput: %.5e .\n", k, total_tag_count, total_tag_time, ((double)total_tag_count) / total_tag_time); } @@ -448,12 +448,12 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) { printf("Iteration %" PRIu64 " : Queries: %" PRIu64 - " , Time: %.4f sec. Query throughput in this iteration: " - "%.4f .\n", + " , Time: %.5e sec. Query throughput in this iteration: " + "%.5e .\n", k, n_query * n_attr, step_elapse, (double)(n_query * n_attr) / step_elapse); printf("Overall %" PRIu64 " : Queries: %" PRIu64 - " , Time: %.4f sec. Overall query throughput: " - "%.4f .\n", + " , Time: %.5e sec. Overall query throughput: " + "%.5e .\n", k, total_query_count, total_query_time, ((double)total_query_count) / total_query_time); } @@ -468,13 +468,13 @@ main(int argc, char *argv[]) if (my_rank == 0) { printf("Final Report: \n"); - printf("[Final Report 1] Servers: %" PRIu64 " , Clients: %" PRIu64 " , C/S ratio: %.4f \n", n_servers, + printf("[Final Report 1] Servers: %" PRIu64 " , Clients: %" PRIu64 " , C/S ratio: %.5e \n", n_servers, n_clients, (double)n_clients / (double)n_servers); printf("[Final Report 2] Iterations: %" PRIu64 " , Objects: %" PRIu64 " , Tags/Object: %" PRIu64 " , Queries/Iteration: " "%" PRIu64 " , \n", k, total_object_count, n_attr, n_query); - printf("[Final Report 3] Object throughput: %.4f , Tag Throughput: %.4f , Query Throughput: %.4f ,", + printf("[Final Report 3] Object throughput: %.5e , Tag Throughput: %.5e , Query Throughput: %.5e ,", (double)total_object_count / total_object_time, (double)(total_object_count * n_attr) / total_tag_time, (double)(total_query_count * n_attr) / total_query_time); diff --git a/src/tests/kvtag_query_scale.c b/src/tests/kvtag_query_scale.c index 80ca2a07c..a42cd5a5a 100644 --- a/src/tests/kvtag_query_scale.c +++ b/src/tests/kvtag_query_scale.c @@ -175,7 +175,7 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) - printf("Total time to query %d objects with tag: %.4f\n", ntotal, total_time); + printf("Total time to query %d objects with tag: %.5e\n", ntotal, total_time); fflush(stdout); } diff --git a/src/tests/list_all.c b/src/tests/list_all.c index 66a282030..508d23db6 100644 --- a/src/tests/list_all.c +++ b/src/tests/list_all.c @@ -187,7 +187,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to create %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to create %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/obj_lock.c b/src/tests/obj_lock.c index bf7b365cc..8a3ea71d3 100644 --- a/src/tests/obj_lock.c +++ b/src/tests/obj_lock.c @@ -118,7 +118,7 @@ main(int argc, char **argv) total_lock_overhead = elapsed / 1000000.0; if (rank == 0) { - printf("Total lock overhead : %.6f\n", total_lock_overhead); + printf("Total lock overhead : %.5e\n", total_lock_overhead); } #ifdef ENABLE_MPI @@ -142,7 +142,7 @@ main(int argc, char **argv) total_lock_overhead = elapsed / 1000000.0; if (rank == 0) { - printf("Total lock release overhead: %.6f\n", total_lock_overhead); + printf("Total lock release overhead: %.5e\n", total_lock_overhead); } // close object diff --git a/src/tests/obj_map.c b/src/tests/obj_map.c index 88d7a8333..2efae2e43 100644 --- a/src/tests/obj_map.c +++ b/src/tests/obj_map.c @@ -168,7 +168,7 @@ main(int argc, char **argv) ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; - printf("Total map overhead : %.6f\n", ht_total_sec); + printf("Total map overhead : %.5e\n", ht_total_sec); fflush(stdout); gettimeofday(&ht_total_start, 0); @@ -182,7 +182,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; - printf("Total unmap overhead : %.6f\n", ht_total_sec); + printf("Total unmap overhead : %.5e\n", ht_total_sec); // close a container if (PDCcont_close(cont_id) < 0) diff --git a/src/tests/obj_transformation.c b/src/tests/obj_transformation.c index 1e9fe2036..a9e2a23d6 100644 --- a/src/tests/obj_transformation.c +++ b/src/tests/obj_transformation.c @@ -272,7 +272,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to map with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to map with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -321,7 +321,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to lock with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to lock with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -384,7 +384,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to update data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to update data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } #ifdef ENABLE_MPI diff --git a/src/tests/query_data.c b/src/tests/query_data.c index c7c28999f..62112f6ed 100644 --- a/src/tests/query_data.c +++ b/src/tests/query_data.c @@ -142,7 +142,7 @@ main(int argc, char **argv) ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to write data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to write data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/query_vpic.c b/src/tests/query_vpic.c index b41377c8c..5b3a43305 100644 --- a/src/tests/query_vpic.c +++ b/src/tests/query_vpic.c @@ -44,7 +44,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); double get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); printf(" Query results:\n"); if (sel.nhits < 500) @@ -63,12 +63,12 @@ main(void) gettimeofday(&pdc_timer_end, 0); double get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); printf("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { - printf("Error with result %" PRIu64 ": %.4f\n", i, energy_data[i]); + printf("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); } } printf("Verified: all correct!\n"); diff --git a/src/tests/query_vpic_bin_sds1_nopreload.c b/src/tests/query_vpic_bin_sds1_nopreload.c index f741f1634..c2ce6e8d2 100644 --- a/src/tests/query_vpic_bin_sds1_nopreload.c +++ b/src/tests/query_vpic_bin_sds1_nopreload.c @@ -79,7 +79,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -95,7 +95,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); fflush(stdout); } diff --git a/src/tests/query_vpic_bin_sds1_preload.c b/src/tests/query_vpic_bin_sds1_preload.c index eff619513..b6e3dd5a3 100644 --- a/src/tests/query_vpic_bin_sds1_preload.c +++ b/src/tests/query_vpic_bin_sds1_preload.c @@ -92,7 +92,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -108,7 +108,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); fflush(stdout); } diff --git a/src/tests/query_vpic_bin_sds_nopreload.c b/src/tests/query_vpic_bin_sds_nopreload.c index 2e6cea44f..d4a89dd44 100644 --- a/src/tests/query_vpic_bin_sds_nopreload.c +++ b/src/tests/query_vpic_bin_sds_nopreload.c @@ -98,7 +98,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -114,7 +114,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); fflush(stdout); } diff --git a/src/tests/query_vpic_bin_sds_preload.c b/src/tests/query_vpic_bin_sds_preload.c index 940bee35d..c0606f66d 100644 --- a/src/tests/query_vpic_bin_sds_preload.c +++ b/src/tests/query_vpic_bin_sds_preload.c @@ -94,7 +94,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -110,7 +110,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); fflush(stdout); } diff --git a/src/tests/query_vpic_exyz_nopreload.c b/src/tests/query_vpic_exyz_nopreload.c index b03167529..d3593b90a 100644 --- a/src/tests/query_vpic_exyz_nopreload.c +++ b/src/tests/query_vpic_exyz_nopreload.c @@ -111,7 +111,7 @@ main(int argc, char **argv) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -127,7 +127,7 @@ main(int argc, char **argv) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] < energy_lo) { diff --git a/src/tests/query_vpic_exyz_preload.c b/src/tests/query_vpic_exyz_preload.c index 0fe32d6e5..80313d58d 100644 --- a/src/tests/query_vpic_exyz_preload.c +++ b/src/tests/query_vpic_exyz_preload.c @@ -95,7 +95,7 @@ main(int argc, char **argv) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Query result in (%" PRIu64 " hits):\n", sel.nhits); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -109,7 +109,7 @@ main(int argc, char **argv) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); fflush(stdout); } diff --git a/src/tests/query_vpic_multi.c b/src/tests/query_vpic_multi.c index 3bb013769..4704b3e1d 100644 --- a/src/tests/query_vpic_multi.c +++ b/src/tests/query_vpic_multi.c @@ -50,7 +50,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Querying Energy in [%.2f, %.2f]\n", energy_lo0, energy_hi0); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -62,12 +62,12 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); printf("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { - printf("Error with result %" PRIu64 ": %.4f\n", i, energy_data[i]); + printf("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); } } printf("Verified: all correct!\n"); diff --git a/src/tests/query_vpic_multi_nopreload.c b/src/tests/query_vpic_multi_nopreload.c index d8debbbbb..dc0da4494 100644 --- a/src/tests/query_vpic_multi_nopreload.c +++ b/src/tests/query_vpic_multi_nopreload.c @@ -50,7 +50,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Querying Energy in [%.2f, %.2f]\n", energy_lo0, energy_hi0); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -62,12 +62,12 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); printf("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { - printf("Error with result %" PRIu64 ": %.4f\n", i, energy_data[i]); + printf("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); } } printf("Verified: all correct!\n"); diff --git a/src/tests/query_vpic_multi_nopreload1.c b/src/tests/query_vpic_multi_nopreload1.c index 1cd7c56b1..f2c73f68b 100644 --- a/src/tests/query_vpic_multi_nopreload1.c +++ b/src/tests/query_vpic_multi_nopreload1.c @@ -50,7 +50,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Querying Energy in [%.2f, %.2f]\n", energy_lo0, energy_hi0); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -62,12 +62,12 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); printf("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { - printf("Error with result %" PRIu64 ": %.4f\n", i, energy_data[i]); + printf("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); } } printf("Verified: all correct!\n"); diff --git a/src/tests/query_vpic_multi_preload.c b/src/tests/query_vpic_multi_preload.c index 3bb013769..4704b3e1d 100644 --- a/src/tests/query_vpic_multi_preload.c +++ b/src/tests/query_vpic_multi_preload.c @@ -50,7 +50,7 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); printf("Querying Energy in [%.2f, %.2f]\n", energy_lo0, energy_hi0); - printf("Get selection time: %.4f\n", get_sel_time); + printf("Get selection time: %.5e\n", get_sel_time); if (sel.nhits > 0) { energy_data = (float *)calloc(sel.nhits, sizeof(float)); @@ -62,12 +62,12 @@ main(void) gettimeofday(&pdc_timer_end, 0); get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); - printf("Get data time: %.4f\n", get_data_time); + printf("Get data time: %.5e\n", get_data_time); printf("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); for (i = 0; i < sel.nhits; i++) { if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { - printf("Error with result %" PRIu64 ": %.4f\n", i, energy_data[i]); + printf("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); } } printf("Verified: all correct!\n"); diff --git a/src/tests/read_obj.c b/src/tests/read_obj.c index aa1fc3c3c..e688219f3 100644 --- a/src/tests/read_obj.c +++ b/src/tests/read_obj.c @@ -208,7 +208,7 @@ main(int argc, char **argv) write_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); if (rank == 0) { - printf("Time to process write data with %d ranks: %.6f\n", size, write_time); + printf("Time to process write data with %d ranks: %.5e\n", size, write_time); fflush(stdout); } @@ -259,7 +259,7 @@ main(int argc, char **argv) write_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); if (rank == 0) { - printf("Time to process read data with %d ranks: %.6f\n", size, write_time); + printf("Time to process read data with %d ranks: %.5e\n", size, write_time); fflush(stdout); } diff --git a/src/tests/search_obj.c b/src/tests/search_obj.c index 2701d729d..465a48b6c 100644 --- a/src/tests/search_obj.c +++ b/src/tests/search_obj.c @@ -233,7 +233,7 @@ main(int argc, char **argv) ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("searched %10d ... %.2f\n", i * size, ht_total_sec); + printf("searched %10d ... %.5e\n", i * size, ht_total_sec); fflush(stdout); } @@ -251,8 +251,8 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - /* printf("Time to full query %d obj/rank with %d ranks: %.6f\n\n\n", count, size, ht_total_sec); */ - printf("Time to partial query %d obj/rank with %d ranks: %.6f\n\n\n", count, size, ht_total_sec); + /* printf("Time to full query %d obj/rank with %d ranks: %.5e\n\n\n", count, size, ht_total_sec); */ + printf("Time to partial query %d obj/rank with %d ranks: %.5e\n\n\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/search_obj_scale.c b/src/tests/search_obj_scale.c index b964c6bc2..88506ffc7 100644 --- a/src/tests/search_obj_scale.c +++ b/src/tests/search_obj_scale.c @@ -180,7 +180,7 @@ main(int argc, char **argv) ht_total_end.tv_usec - ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; - printf("%10d queried ... %.2fs\n", i * size, ht_total_sec); + printf("%10d queried ... %.5es\n", i * size, ht_total_sec); fflush(stdout); } } @@ -193,7 +193,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to create %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to create %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/stat_obj.c b/src/tests/stat_obj.c index da8a5f927..6a068edc7 100644 --- a/src/tests/stat_obj.c +++ b/src/tests/stat_obj.c @@ -207,7 +207,7 @@ main(int argc, char **argv) ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("stated %10d ... %.2f\n", i * size, ht_total_sec); + printf("stated %10d ... %.5e\n", i * size, ht_total_sec); fflush(stdout); } @@ -225,7 +225,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to stat %d obj/rank with %d ranks: %.6f\n", count, size, ht_total_sec); + printf("Time to stat %d obj/rank with %d ranks: %.5e\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/update_obj.c b/src/tests/update_obj.c index a589725ce..b63082361 100644 --- a/src/tests/update_obj.c +++ b/src/tests/update_obj.c @@ -211,7 +211,7 @@ main(int argc, char **argv) ht_total_end.tv_usec - ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; - printf("updated %10d ... %.2f\n", i * size, ht_total_sec); + printf("updated %10d ... %.5e\n", i * size, ht_total_sec); fflush(stdout); } } @@ -225,7 +225,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to update %d obj/rank with %d ranks: %.6f\n\n\n", count, size, ht_total_sec); + printf("Time to update %d obj/rank with %d ranks: %.5e\n\n\n", count, size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/vpicio.c b/src/tests/vpicio.c index 1a6692628..081376f43 100644 --- a/src/tests/vpicio.c +++ b/src/tests/vpicio.c @@ -241,7 +241,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Obj create time: %.2f\n", t1 - t0); + printf("Obj create time: %.5e\n", t1 - t0); } #endif @@ -290,7 +290,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("Transfer create time: %.2f\n", t0 - t1); + printf("Transfer create time: %.5e\n", t0 - t1); } #endif @@ -339,7 +339,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Transfer start time: %.2f\n", t1 - t0); + printf("Transfer start time: %.5e\n", t1 - t0); } #endif @@ -388,7 +388,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("Transfer wait time: %.2f\n", t0 - t1); + printf("Transfer wait time: %.5e\n", t0 - t1); } #endif @@ -438,7 +438,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Transfer close time: %.2f\n", t1 - t0); + printf("Transfer close time: %.5e\n", t1 - t0); } #endif diff --git a/src/tests/vpicio_mts.c b/src/tests/vpicio_mts.c index 110febbfd..eec66215b 100644 --- a/src/tests/vpicio_mts.c +++ b/src/tests/vpicio_mts.c @@ -258,7 +258,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Obj create time: %.2f\n", t1 - t0); + printf("Obj create time: %.5e\n", t1 - t0); } #endif @@ -309,7 +309,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("Transfer create time: %.2f\n", t0 - t1); + printf("Transfer create time: %.5e\n", t0 - t1); } #endif @@ -358,7 +358,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Transfer start time: %.2f\n", t1 - t0); + printf("Transfer start time: %.5e\n", t1 - t0); } #endif @@ -407,7 +407,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("Transfer wait time: %.2f\n", t0 - t1); + printf("Transfer wait time: %.5e\n", t0 - t1); } #endif @@ -456,7 +456,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) { - printf("Transfer close time: %.2f\n", t1 - t0); + printf("Transfer close time: %.5e\n", t1 - t0); } #endif @@ -498,7 +498,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("Obj close time: %.2f\n", t0 - t1); + printf("Obj close time: %.5e\n", t0 - t1); } #endif if (i != steps - 1) { diff --git a/src/tests/vpicio_v2.c b/src/tests/vpicio_v2.c index ee3f3512a..89839352b 100644 --- a/src/tests/vpicio_v2.c +++ b/src/tests/vpicio_v2.c @@ -299,7 +299,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to map with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to map with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -348,7 +348,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to lock with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to lock with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } @@ -408,7 +408,7 @@ main(int argc, char **argv) ht_total_start.tv_usec; ht_total_sec = ht_total_elapsed / 1000000.0; if (rank == 0) { - printf("Time to update data with %d ranks: %.6f\n", size, ht_total_sec); + printf("Time to update data with %d ranks: %.5e\n", size, ht_total_sec); fflush(stdout); } diff --git a/src/tests/write_obj_shared.c b/src/tests/write_obj_shared.c index 9fa01107f..d8796bb61 100644 --- a/src/tests/write_obj_shared.c +++ b/src/tests/write_obj_shared.c @@ -209,7 +209,7 @@ main(int argc, char **argv) write_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); if (rank == 0) { - printf("Time to lock and release data with %d ranks: %.6f\n", size, write_time); + printf("Time to lock and release data with %d ranks: %.5e\n", size, write_time); fflush(stdout); } done: From e30f7b7d5c5360e23ac1b5849620f8305cd5d091 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Thu, 15 Jun 2023 13:57:46 -0400 Subject: [PATCH 190/216] llsm_importer (#1) formatter on llsm_importer --- tools/CMakeLists.txt | 2 ++ tools/llsm/parallelReadTiff.c | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index e15e21992..b14402393 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -102,6 +102,8 @@ add_library(cjson cjson/cJSON.c) # endforeach(program) +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -fopenmp -DNDEBUG") + # Find LibTIFF option(USE_LIB_TIFF "Enable LibTiff." ON) if(USE_LIB_TIFF) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 4da885e61..c81928584 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -1,7 +1,7 @@ #include "parallelReadTiff.h" #include "tiffio.h" -// #define ENABLE_OPENMP +#define ENABLE_OPENMP #ifdef ENABLE_OPENMP #include "omp.h" @@ -543,7 +543,7 @@ readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char *fileName, TIFFClose(tif); lseek(fd, offset, SEEK_SET); uint64_t bytes = bits / 8; - //#pragma omp parallel for + // #pragma omp parallel for /* for(uint64_t i = 0; i < z; i++){ uint64_t cOffset = x*y*bytes*i; @@ -784,7 +784,7 @@ void parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, image_info_t **image_info) { - uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 0, is_imageJ = 0, imageJ_Z = 0; + uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 1, is_imageJ = 0, imageJ_Z = 0; get_tiff_info(fileName, strip_range, &x, &y, &z, &bits, &startSlice, &stripeSize, &is_imageJ, &imageJ_Z); From 917b4f5fa8f2b0f6e1d5d344fd55dd503a8c9a0d Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Thu, 15 Jun 2023 14:07:09 -0400 Subject: [PATCH 191/216] Tiff Parallel Reader sync to latest version (#89) * remove unnecessary install block from CMakeLists.txt * update output * Revert "update output" This reverts commit fe1f8b44995bc0dabd3b957e1032c2da26f56fdd. * build kvtag_add_get_scale * comment off free * update code * 1. kvtag_scale_add_get added \n 2. uint64_t support for obj/tag/query count \n 3. moving work assigning block downwards right before creating objects \n 4. everything is tested working * do while loop added, tested with 1m object and works * 1m objects test works, 10m object test fail as the original also fails * add new executable to test set * enlarge PDC_SERVER_ID_INTERVAL * update code * update console args * add p search test * add console arg for changing number of attributes per object * free allocated memory * fix query count issue * fix attr length definition * code refactored * code refactored * code refactored * code refactored * code refactored * code refactored * fix data type * fix data type * fix data type * add client side statistics * add client side statistics * fix format * clang formatter * update CMake * update CMake * update CMake * free allocated memory properly * clang format * clang format * clang-format-10 * change file name * address review comments * update llsm importer * update llsm importer * update server checkpoint intervals * update gitignore * adding job scripts * adding one debugging msg * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update output for uint64_t * add scripts * update output for uint64_t * update output for uint64_t * update output for uint64_t * update scripts * update scripts * delete debugging message * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * update tag names * update tag names * update query startingpos * update query startingpos * update job scripts * add progressive timing for kvtag_add_get_scale * fix iteration count in final report * update job scripts and benckmark program * update message format * update message format * update message format * update message format * clang format * update job scripts * comment off object/container close procedure in benchmark to save node hours * change the max number of object to 1M * change the max length of attribute value * change the max length of attribute value * llsm tiff import test * llsm tiff import test * llsm tiff import test * llsm tiff import test * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update cmake and llsm_importer * update cmake and llsm_importer * close if in cmake * cmake fix tiff * cmake policy to suppress warning * add pdc include dir * update code * update code * update code * update code * update code * update code * update array generating method * update array generating method * update array generating method * update array generating method * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * fix return type * fix return type * add timing * add timing * fix output * llsm tiff importer 1st version: read csv and import tiff files to PDC, adding metadata available in CSV files and TIFF loader * fix vairable name * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * add scripts * add scripts * add scripts * debugging for nonMPI program * debugging for nonMPI program * debugging for nonMPI program * clang format, without PDC, everything works perfectly. program fails at PDC init stage where PDCprop_create(PDC_CONT_CREATE, pdc) is being created * enable MPI * enable MPI * enlarge BCase size * enlarge BCase size * enlarge BCase size * resolve bcast count * llsm data path in script * llsm data path in script * update csv reader * update csv reader * update csv reader * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * enlarge max write * update pdc * update pdc * update pdc * update pdc * update pdc_import.c * update pdc_import.c * update pdc_export.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update tools/cmake * clang format * clang format * added a tutorial for llsm_importer * added a tutorial for llsm_importer * make sure the line feed is included for string attribute * update timing for overall completion time * update formatting * llsm_importer (#1) formatter on llsm_importer --------- Co-authored-by: Houjun Tang --- tools/CMakeLists.txt | 2 ++ tools/llsm/parallelReadTiff.c | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index e15e21992..b14402393 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -102,6 +102,8 @@ add_library(cjson cjson/cJSON.c) # endforeach(program) +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -fopenmp -DNDEBUG") + # Find LibTIFF option(USE_LIB_TIFF "Enable LibTiff." ON) if(USE_LIB_TIFF) diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index 4da885e61..c81928584 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -1,7 +1,7 @@ #include "parallelReadTiff.h" #include "tiffio.h" -// #define ENABLE_OPENMP +#define ENABLE_OPENMP #ifdef ENABLE_OPENMP #include "omp.h" @@ -543,7 +543,7 @@ readTiffParallelImageJ(uint64_t x, uint64_t y, uint64_t z, const char *fileName, TIFFClose(tif); lseek(fd, offset, SEEK_SET); uint64_t bytes = bits / 8; - //#pragma omp parallel for + // #pragma omp parallel for /* for(uint64_t i = 0; i < z; i++){ uint64_t cOffset = x*y*bytes*i; @@ -784,7 +784,7 @@ void parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, image_info_t **image_info) { - uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 0, is_imageJ = 0, imageJ_Z = 0; + uint64_t x = 1, y = 1, z = 1, bits = 1, startSlice = 0, stripeSize = 1, is_imageJ = 0, imageJ_Z = 0; get_tiff_info(fileName, strip_range, &x, &y, &z, &bits, &startSlice, &stripeSize, &is_imageJ, &imageJ_Z); From 8448043e83094c02da23cd1779b5ba9b0dca70e2 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Fri, 16 Jun 2023 15:05:31 -0700 Subject: [PATCH 192/216] Update .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27e4d751c..39f940487 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -162,7 +162,7 @@ perlmutter-metrics: tags: - perlmutter variables: - PDC_N_NODES: 4 + PDC_N_NODES: 64 PDC_N_CLIENTS: 127 SCHEDULER_PARAMETERS: "-A m1248 --qos=debug --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00" SUPERCOMPUTER: "perlmutter" From 10e0bb1d37a03e428b803e557d753aecaf263833 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Mon, 19 Jun 2023 17:11:10 -0400 Subject: [PATCH 193/216] add type for kvtag structure (#2) * upate metadata type system * update serde framework to coupe with the new data type system * replace unnecessary data types * adding type for pdc_kvtag_t, all occurances are fixed * update new commons CMake for publishing commons * commons compilation passed * compiled --- CMakeLists.txt | 3 +- docs/readme.md | 154 ++--- docs/source/api.rst | 37 +- src/api/CMakeLists.txt | 27 +- src/api/include/pdc_public.h | 78 --- src/api/pdc_client_connect.c | 18 +- src/api/pdc_obj/include/pdc_cont.h | 4 +- src/api/pdc_obj/include/pdc_obj.h | 6 +- src/api/pdc_obj/include/pdc_prop_pkg.h | 7 +- src/api/pdc_obj/pdc_dt_conv.c | 8 +- src/api/pdc_obj/pdc_obj.c | 1 + src/api/profiling/CMakeLists.txt | 120 ---- src/api/profiling/include/pdc_hashtab.h | 198 ------- src/api/profiling/include/pdc_stack_ops.h | 70 --- src/api/profiling/pdc_hashtab.c | 540 ------------------ src/api/profiling/pdc_stack_ops.c | 264 --------- src/commons/CMakeLists.txt | 227 ++++++++ src/commons/generic/include/pdc_generic.h | 219 +++++++ src/commons/serde/include/pdc_serde.h | 155 +++++ src/commons/serde/pdc_serde.c | 346 +++++++++++ src/server/CMakeLists.txt | 5 +- src/server/include/pdc_client_server_common.h | 5 + src/server/pdc_client_server_common.c | 6 +- src/server/pdc_server.c | 7 +- .../pdc_server_region_request_handler.h | 4 +- src/tests/cont_tags.c | 17 +- src/tests/kvtag_add_get.c | 35 +- src/tests/kvtag_add_get_benchmark.c | 5 +- src/tests/kvtag_add_get_scale.c | 8 +- src/tests/kvtag_get.c | 15 +- src/tests/kvtag_query.c | 11 +- src/tests/kvtag_query_scale.c | 4 +- src/tests/obj_tags.c | 17 +- src/tests/pdc_transforms_lib.c | 35 +- src/utils/include/pdc_id_pkg.h | 57 -- src/utils/include/pdc_linkedlist.h | 120 ---- src/utils/include/pdc_malloc.h | 59 -- src/utils/include/pdc_private.h | 202 ------- src/utils/include/pdc_timing.h | 193 ------- src/utils/pdc_interface.c | 2 + src/utils/pdc_malloc.c | 76 --- src/utils/pdc_timing.c | 537 ----------------- tools/pdc_export.c | 41 +- tools/pdc_import.c | 9 +- tools/pdc_ls.c | 40 +- 45 files changed, 1219 insertions(+), 2773 deletions(-) delete mode 100644 src/api/include/pdc_public.h delete mode 100644 src/api/profiling/CMakeLists.txt delete mode 100644 src/api/profiling/include/pdc_hashtab.h delete mode 100644 src/api/profiling/include/pdc_stack_ops.h delete mode 100644 src/api/profiling/pdc_hashtab.c delete mode 100644 src/api/profiling/pdc_stack_ops.c create mode 100644 src/commons/CMakeLists.txt create mode 100644 src/commons/generic/include/pdc_generic.h create mode 100644 src/commons/serde/include/pdc_serde.h create mode 100644 src/commons/serde/pdc_serde.c delete mode 100644 src/utils/include/pdc_id_pkg.h delete mode 100644 src/utils/include/pdc_linkedlist.h delete mode 100644 src/utils/include/pdc_malloc.h delete mode 100644 src/utils/include/pdc_private.h delete mode 100644 src/utils/include/pdc_timing.h delete mode 100644 src/utils/pdc_malloc.c delete mode 100644 src/utils/pdc_timing.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e353dbc2..5e7b65d94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -257,7 +257,7 @@ endif() option(PDC_ENABLE_LUSTRE "Enable Lustre." OFF) if(PDC_ENABLE_LUSTRE) set(ENABLE_LUSTRE 1) - set(PDC_LUSTRE_TOTAL_OST "248" CACHE STRING "Number of Lustre OSTs") + set(PDC_LUSTRE_TOTAL_OST "256" CACHE STRING "Number of Lustre OSTs") endif() #----------------------------------------------------------------------------- @@ -384,6 +384,7 @@ configure_file( #----------------------------------------------------------------------------- # Source #----------------------------------------------------------------------------- +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/commons) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/api) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/server) diff --git a/docs/readme.md b/docs/readme.md index df19eba94..74be2d0e4 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,58 +1,67 @@ # PDC Documentations - + [PDC user APIs](#pdc-user-apis) - - [PDC general APIs](#pdc-general-apis) - - [PDC container APIs](#pdc-container-apis) - - [PDC object APIs](#pdc-object-apis) - - [PDC region APIs](#pdc-region-apis) - - [PDC property APIs](#pdc-property-apis) - - [PDC query APIs](#pdc-query-apis) - + [PDC data types](#PDC-type-categories) - - [Basic types](#basic-types) - - [Histogram structure](#histogram-structure) - - [Container info](#container-info) - - [Container life time](#container-life-time) - - [Object property](#object-property) - - [Object info](#object-info) - - [Object structure](#object-structure) - - [Region info](#region-info) - - [Access type](#access-type) - - [Transfer request status](#transfer-request-status) - - [Query operators](#query-operators) - - [Query structures](#query-structures) - - [Selection structure](#selection-structure) - + [Developers notes](#developers-notes) - - [How to implement an RPC from client to server](#how-to-implement-an-rpc-from-client-to-server) - - [PDC Server metadata overview](#pdc-server-metadata-overview) - + [PDC metadata structure](#pdc-metadata-structure) - + [Metadata operations at client side](#metadata-operations-at-client-side) - - [PDC metadata management strategy](#pdc-metadata-management-strategy) - + [Managing metadata and data by the same server](#managing-metadata-and-data-by-the-same-server) - + [Separate metadata server from data server](#separate-metadata-server-from-data-server) - + [Static object region mappings](#static-object-region-mappings) - + [Dynamic object region mappings](#dynamic-object-region-mappings) - - [PDC metadata management implementation](#pdc-metadata-management-implementation) - + [Create metadata](#create-metadata) - + [Binding metadata to object](#binding-metadata-to-object) - + [Register object metadata at metadata server](#register-object-metadata-at-metadata-server) - + [Retrieve metadata from metadata server](#retrieve-metadata-from-metadata-server) - + [Object metadata at client](#object-metadata-at-client) - + [Metadata at data server](#metadata-at-data-server) - + [Object metadata update](#object-metadata-update) - + [Object region metadata](#object-region-metadata) - + [Metadata checkpoint](#object-metadata-update) - - [Region transfer request at client](#region-transfer-request-at-client) - + [Region transfer request create and close](#region-transfer-request-create-and-close) - + [Region transfer request start](#region-transfer-request-start) - + [Region transfer request wait](#region-transfer-request-wait) - - [Region transfer request at server](#region-transfer-request-at-server) - + [Server region transfer request RPC](#server-region-transfer-request-rpc) - - [Server nonblocking control](#server-nonblocking-control) - - [Server region transfer request start](#server-region-transfer-request-start) - - [Server region transfer request wait](#server-region-transfer-request-wait) - + [Server region storage](#server-region-storage) - - [Storage by file offset](#storage-by-file-offset) - - [Storage by region](#storage-by-region) - - [Open tasks for PDC](#open-tasks-for-pdc) +- [PDC Documentations](#pdc-documentations) +- [PDC user APIs](#pdc-user-apis) + - [PDC general APIs](#pdc-general-apis) + - [PDC container APIs](#pdc-container-apis) + - [PDC object APIs](#pdc-object-apis) + - [PDC region APIs](#pdc-region-apis) + - [PDC property APIs](#pdc-property-apis) + - [PDC query APIs](#pdc-query-apis) + - [PDC hist APIs](#pdc-hist-apis) +- [PDC Data types](#pdc-data-types) + - [Basic types](#basic-types) + - [region transfer partition type](#region-transfer-partition-type) + - [Object consistency semantics type](#object-consistency-semantics-type) + - [Histogram structure](#histogram-structure) + - [Container info](#container-info) + - [Container life time](#container-life-time) + - [Object property public](#object-property-public) + - [Object property](#object-property) + - [Object info](#object-info) + - [Object structure](#object-structure) + - [Region info](#region-info) + - [Access type](#access-type) + - [Transfer request status](#transfer-request-status) + - [Query operators](#query-operators) + - [Query structures](#query-structures) + - [Selection structure](#selection-structure) +- [Developers notes](#developers-notes) + - [How to implement an RPC from client to server](#how-to-implement-an-rpc-from-client-to-server) + - [PDC Server metadata overview](#pdc-server-metadata-overview) + - [PDC metadata structure](#pdc-metadata-structure) + - [Metadata operations at client side](#metadata-operations-at-client-side) + - [PDC metadata management strategy](#pdc-metadata-management-strategy) + - [Managing metadata and data by the same server](#managing-metadata-and-data-by-the-same-server) + - [Separate metadata server from data server](#separate-metadata-server-from-data-server) + - [Static object region mappings](#static-object-region-mappings) + - [Dynamic object region mappings](#dynamic-object-region-mappings) + - [PDC metadata management implementation](#pdc-metadata-management-implementation) + - [Create metadata](#create-metadata) + - [Binding metadata to object](#binding-metadata-to-object) + - [Register object metadata at metadata server](#register-object-metadata-at-metadata-server) + - [Retrieve metadata from metadata server](#retrieve-metadata-from-metadata-server) + - [Object metadata at client](#object-metadata-at-client) + - [Metadata at data server](#metadata-at-data-server) + - [Object metadata update](#object-metadata-update) + - [Object region metadata](#object-region-metadata) + - [Metadata checkpoint](#metadata-checkpoint) + - [Region transfer request at client](#region-transfer-request-at-client) + - [Region transfer request create and close](#region-transfer-request-create-and-close) + - [Region transfer request start](#region-transfer-request-start) + - [Region transfer request wait](#region-transfer-request-wait) + - [Region transfer request at server](#region-transfer-request-at-server) + - [Server region transfer request RPC](#server-region-transfer-request-rpc) + - [Server nonblocking control](#server-nonblocking-control) + - [Server region transfer request start](#server-region-transfer-request-start) + - [Server region transfer request wait](#server-region-transfer-request-wait) + - [Server region storage](#server-region-storage) + - [Storage by file offset](#storage-by-file-offset) + - [Storage by region](#storage-by-region) + - [Open tasks for PDC](#open-tasks-for-pdc) + - [Replacing individual modules with efficient Hash table data structures](#replacing-individual-modules-with-efficient-hash-table-data-structures) + - [Restarting pdc\_server.exe with different numbers of servers](#restarting-pdc_serverexe-with-different-numbers-of-servers) + - [Fast region search mechanisms](#fast-region-search-mechanisms) + - [Merge overlapping regions](#merge-overlapping-regions) # PDC user APIs ## PDC general APIs + pdcid_t PDCinit(const char *pdc_name) @@ -683,21 +692,28 @@ ## Basic types ``` typedef enum { - PDC_UNKNOWN = -1, /* error */ - PDC_INT = 0, /* integer types */ - PDC_FLOAT = 1, /* floating-point types */ - PDC_DOUBLE = 2, /* double types */ - PDC_CHAR = 3, /* character types */ - PDC_COMPOUND = 4, /* compound types */ - PDC_ENUM = 5, /* enumeration types */ - PDC_ARRAY = 6, /* Array types */ - PDC_UINT = 7, /* unsigned integer types */ - PDC_INT64 = 8, /* 64-bit integer types */ - PDC_UINT64 = 9, /* 64-bit unsigned integer types */ - PDC_INT16 = 10, - PDC_INT8 = 11, - NCLASSES = 12 /* this must be last */ - } pdc_var_type_t; + PDC_UNKNOWN = -1, /* error */ + PDC_INT = 0, /* integer types (identical to int32_t) */ + PDC_FLOAT = 1, /* floating-point types */ + PDC_DOUBLE = 2, /* double types */ + PDC_CHAR = 3, /* character types */ + PDC_STRING = 4, /* string types */ + PDC_BOOLEAN = 5, /* boolean types */ + PDC_SHORT = 6, /* short types */ + PDC_UINT = 7, /* unsigned integer types (identical to uint32_t) */ + PDC_INT64 = 8, /* 64-bit integer types */ + PDC_UINT64 = 9, /* 64-bit unsigned integer types */ + PDC_INT16 = 10, /* 16-bit integer types */ + PDC_INT8 = 11, /* 8-bit integer types */ + PDC_UINT8 = 12, /* 8-bit unsigned integer types */ + PDC_UINT16 = 13, /* 16-bit unsigned integer types */ + PDC_INT32 = 14, /* 32-bit integer types */ + PDC_UINT32 = 15, /* 32-bit unsigned integer types */ + PDC_LONG = 16, /* long types */ + PDC_VOID_PTR = 17, /* void pointer type */ + PDC_SIZE_T = 18, /* size_t type */ + PDC_TYPE_COUNT = 19 /* this is the number of var types and has to be the last */ + } pdc_c_var_type_t; ``` ## region transfer partition type ``` diff --git a/docs/source/api.rst b/docs/source/api.rst index e9b1e6567..ab058f10a 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -471,21 +471,28 @@ Basic types .. code-block:: c typedef enum { - PDC_UNKNOWN = -1, /* error */ - PDC_INT = 0, /* integer types */ - PDC_FLOAT = 1, /* floating-point types */ - PDC_DOUBLE = 2, /* double types */ - PDC_CHAR = 3, /* character types */ - PDC_COMPOUND = 4, /* compound types */ - PDC_ENUM = 5, /* enumeration types */ - PDC_ARRAY = 6, /* Array types */ - PDC_UINT = 7, /* unsigned integer types */ - PDC_INT64 = 8, /* 64-bit integer types */ - PDC_UINT64 = 9, /* 64-bit unsigned integer types */ - PDC_INT16 = 10, - PDC_INT8 = 11, - NCLASSES = 12 /* this must be last */ - } pdc_var_type_t; + PDC_UNKNOWN = -1, /* error */ + PDC_INT = 0, /* integer types (identical to int32_t) */ + PDC_FLOAT = 1, /* floating-point types */ + PDC_DOUBLE = 2, /* double types */ + PDC_CHAR = 3, /* character types */ + PDC_STRING = 4, /* string types */ + PDC_BOOLEAN = 5, /* boolean types */ + PDC_SHORT = 6, /* short types */ + PDC_UINT = 7, /* unsigned integer types (identical to uint32_t) */ + PDC_INT64 = 8, /* 64-bit integer types */ + PDC_UINT64 = 9, /* 64-bit unsigned integer types */ + PDC_INT16 = 10, /* 16-bit integer types */ + PDC_INT8 = 11, /* 8-bit integer types */ + PDC_UINT8 = 12, /* 8-bit unsigned integer types */ + PDC_UINT16 = 13, /* 16-bit unsigned integer types */ + PDC_INT32 = 14, /* 32-bit integer types */ + PDC_UINT32 = 15, /* 32-bit unsigned integer types */ + PDC_LONG = 16, /* long types */ + PDC_VOID_PTR = 17, /* void pointer type */ + PDC_SIZE_T = 18, /* size_t type */ + PDC_TYPE_COUNT = 19 /* this is the number of var types and has to be the last */ + } pdc_c_var_type_t; diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index 6d4e0b722..7ef5ec186 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -1,7 +1,7 @@ #------------------------------------------------------------------------------ # Include source and build directories #------------------------------------------------------------------------------ -set( LOCAL_INCLUDE_DIR +set(LOCAL_INCLUDE_DIR ${PDC_INCLUDES_BUILD_TIME} ${PROJECT_BINARY_DIR} ${PDC_SOURCE_DIR} @@ -32,9 +32,9 @@ include_directories( # External dependencies #------------------------------------------------------------------------------ # profiling -#set(PDC_EXT_LIB_DEPENDENCIES pdcprof ${PDC_EXT_LIB_DEPENDENCIES}) -set(PDC_EXT_INCLUDE_DEPENDENCIES ${CMAKE_CURRENT_SOURCE_DIR}/profiling) -set(PDC_EXPORTED_LIBS pdcprof) +# set(PDC_EXT_LIB_DEPENDENCIES pdcprof ${PDC_EXT_LIB_DEPENDENCIES}) +# set(PDC_EXT_INCLUDE_DEPENDENCIES ${CMAKE_CURRENT_SOURCE_DIR}/profiling) +# set(PDC_EXPORTED_LIBS pdcprof) # Mercury find_package(MERCURY REQUIRED) @@ -87,22 +87,24 @@ set(PDC_SRCS ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_cache.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c - ${PDC_SOURCE_DIR}/src/utils/pdc_timing.c - ${PDC_SOURCE_DIR}/src/utils/pdc_malloc.c ${PDC_SOURCE_DIR}/src/utils/pdc_interface.c ${PDC_SOURCE_DIR}/src/utils/pdc_region_utils.c ) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/profiling) + set(PDC_COMMON_INCLUDE_DIRS ${PDC_COMMON_INCLUDE_DIRS}) + # add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/profiling) #------------------------------------------------------------------------------ # Libraries #------------------------------------------------------------------------------ # PDC set(PDC_BUILD_INCLUDE_DEPENDENCIES + ${PDC_COMMON_INCLUDE_DIRS} ${LOCAL_INCLUDE_DIR} ) +message(STATUS "PDC_BUILD_INCLUDE_DEPENDENCIES: ${PDC_BUILD_INCLUDE_DEPENDENCIES}") + add_library(pdc ${PDC_SRCS}) target_include_directories(pdc @@ -110,7 +112,12 @@ target_include_directories(pdc $ ) +message(STATUS "PDC_EXPORTED_LIBS: ${PDC_EXPORTED_LIBS}") +message(STATUS "PDC_EXT_LIB_DEPENDENCIES: ${PDC_EXT_LIB_DEPENDENCIES}") +message(STATUS "PDC_COMMONS_LIBRARIES: ${PDC_COMMONS_LIBRARIES}") + target_link_libraries(pdc + ${PDC_COMMONS_LIBRARIES} ${PDC_EXPORTED_LIBS} ${PDC_EXT_LIB_DEPENDENCIES} -ldl @@ -122,7 +129,7 @@ set(PDC_EXPORTED_LIBS pdc ${PDC_EXPORTED_LIBS}) add_executable(close_server close_server.c ) -target_link_libraries(close_server pdc) +target_link_libraries(close_server pdc ${PDC_COMMON_LIBRARIES}) install( TARGETS @@ -145,6 +152,7 @@ set(PDC_HEADERS ${PDC_SOURCE_DIR}/src/api/pdc_query/include/pdc_query.h ${PDC_SOURCE_DIR}/src/api/pdc_region/include/pdc_region.h ${PDC_SOURCE_DIR}/src/api/pdc_transform/include/pdc_transform.h + ${PDC_SOURCE_DIR}/src/utils/include/pdc_interface.h ${PROJECT_BINARY_DIR}/pdc_config_sys.h ${PROJECT_BINARY_DIR}/pdc_config.h ) @@ -173,9 +181,10 @@ install( #----------------------------------------------------------------------------- # Add Target(s) to CMake Install #----------------------------------------------------------------------------- + install( TARGETS - pdc + pdc EXPORT ${PDC_EXPORTED_TARGETS} LIBRARY DESTINATION ${PDC_INSTALL_LIB_DIR} diff --git a/src/api/include/pdc_public.h b/src/api/include/pdc_public.h deleted file mode 100644 index 8c47976f0..000000000 --- a/src/api/include/pdc_public.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#ifndef PDC_PUBLIC_H -#define PDC_PUBLIC_H - -#include -#include -#include - -/*******************/ -/* Public Typedefs */ -/*******************/ -typedef int perr_t; -typedef uint64_t pdcid_t; -typedef unsigned long long psize_t; -typedef bool pbool_t; - -typedef int PDC_int_t; -typedef float PDC_float_t; -typedef double PDC_double_t; - -typedef enum { - PDC_UNKNOWN = -1, /* error */ - PDC_INT = 0, /* integer types */ - PDC_FLOAT = 1, /* floating-point types */ - PDC_DOUBLE = 2, /* double types */ - PDC_CHAR = 3, /* character types */ - PDC_COMPOUND = 4, /* compound types */ - PDC_ENUM = 5, /* enumeration types */ - PDC_ARRAY = 6, /* Array types */ - PDC_UINT = 7, /* unsigned integer types */ - PDC_INT64 = 8, /* 64-bit integer types */ - PDC_UINT64 = 9, /* 64-bit unsigned integer types */ - PDC_INT16 = 10, - PDC_INT8 = 11, - PDC_UINT8 = 12, - PDC_UINT16 = 13, - NCLASSES = 14 /* this must be last */ -} pdc_var_type_t; - -typedef enum { PDC_PERSIST, PDC_TRANSIENT } pdc_lifetime_t; - -typedef enum { PDC_SERVER_DEFAULT = 0, PDC_SERVER_PER_CLIENT = 1 } pdc_server_selection_t; - -typedef struct pdc_histogram_t { //????????? - pdc_var_type_t dtype; - int nbin; - double incr; - double * range; - uint64_t * bin; -} pdc_histogram_t; - -#define SUCCEED 0 -#define FAIL (-1) - -#endif /* PDC_PUBLIC_H */ diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index bdf967885..b6b969f8a 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -33,6 +33,7 @@ #include "pdc_utlist.h" #include "pdc_id_pkg.h" +#include "pdc_cont_pkg.h" #include "pdc_prop_pkg.h" #include "pdc_obj_pkg.h" #include "pdc_cont.h" @@ -7018,6 +7019,7 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) if (kvtag != NULL && kvtag != NULL && kvtag->size != 0) { in.kvtag.name = kvtag->name; in.kvtag.value = kvtag->value; + in.kvtag.type = kvtag->type; in.kvtag.size = kvtag->size; } else @@ -7061,6 +7063,7 @@ metadata_get_kvtag_rpc_cb(const struct hg_cb_info *callback_info) client_lookup_args->ret = output.ret; client_lookup_args->kvtag->name = strdup(output.kvtag.name); client_lookup_args->kvtag->size = output.kvtag.size; + client_lookup_args->kvtag->type = output.kvtag.type; client_lookup_args->kvtag->value = malloc(output.kvtag.size); memcpy(client_lookup_args->kvtag->value, output.kvtag.value, output.kvtag.size); /* PDC_kvtag_dup(&(output.kvtag), &client_lookup_args->kvtag); */ @@ -7315,10 +7318,12 @@ PDC_Client_query_kvtag_server(uint32_t server_id, const pdc_kvtag_t *kvtag, int if (kvtag->value == NULL) { in.value = " "; + in.type = PDC_STRING; in.size = 1; } else { in.value = kvtag->value; + in.type = kvtag->type; in.size = kvtag->size; } @@ -7576,7 +7581,7 @@ PDCcont_get_objids(pdcid_t cont_id ATTRIBUTE(unused), int *nobj ATTRIBUTE(unused } perr_t -PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_size) +PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, psize_t value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t kvtag; @@ -7585,6 +7590,7 @@ PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_ kvtag.name = tag_name; kvtag.value = (void *)tag_value; + kvtag.type = value_type; kvtag.size = (uint64_t)value_size; ret_value = PDC_add_kvtag(cont_id, &kvtag, 1); @@ -7598,7 +7604,7 @@ PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_ } perr_t -PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, psize_t *value_size) +PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, psize_t *value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t *kvtag = NULL; @@ -7610,6 +7616,7 @@ PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, psize_t *valu PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Error with PDC_get_kvtag", pdc_client_mpi_rank_g); *tag_value = kvtag->value; + *value_type = kvtag->type; *value_size = kvtag->size; done: @@ -7772,7 +7779,7 @@ PDC_Client_del_metadata(pdcid_t obj_id, int is_cont) } perr_t -PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_size) +PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, psize_t value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t kvtag; @@ -7781,6 +7788,7 @@ PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_si kvtag.name = tag_name; kvtag.value = (void *)tag_value; + kvtag.type = value_type; kvtag.size = (uint64_t)value_size; ret_value = PDC_add_kvtag(obj_id, &kvtag, 0); @@ -7793,7 +7801,8 @@ PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_si } perr_t -PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, psize_t *value_size) +PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t *kvtag = NULL; @@ -7805,6 +7814,7 @@ PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, psize_t *value_ PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Error with PDC_get_kvtag", pdc_client_mpi_rank_g); *tag_value = kvtag->value; + *value_type = kvtag->type; *value_size = kvtag->size; done: diff --git a/src/api/pdc_obj/include/pdc_cont.h b/src/api/pdc_obj/include/pdc_cont.h index 844b15425..3a6180b65 100644 --- a/src/api/pdc_obj/include/pdc_cont.h +++ b/src/api/pdc_obj/include/pdc_cont.h @@ -191,7 +191,7 @@ perr_t PDCcont_del(pdcid_t cont_id); * * \return Non-negative on success/Negative on failure */ -perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_size); +perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, psize_t value_size); /** * *********** @@ -203,7 +203,7 @@ perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t * * \return Non-negative on success/Negative on failure */ -perr_t PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, psize_t *value_size); +perr_t PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, psize_t *value_size); /** * Deleta a tag from a container diff --git a/src/api/pdc_obj/include/pdc_obj.h b/src/api/pdc_obj/include/pdc_obj.h index f678adf7f..8ad7a285a 100644 --- a/src/api/pdc_obj/include/pdc_obj.h +++ b/src/api/pdc_obj/include/pdc_obj.h @@ -409,7 +409,8 @@ perr_t PDCobj_del(pdcid_t obj_id); * * \return Non-negative on success/Negative on failure */ -perr_t PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_size); +perr_t PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, + psize_t value_size); /** * Get tag information @@ -421,7 +422,8 @@ perr_t PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t v * * \return Non-negative on success/Negative on failure */ -perr_t PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, psize_t *value_size); +perr_t PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size); /** * Delete a tag from the object diff --git a/src/api/pdc_obj/include/pdc_prop_pkg.h b/src/api/pdc_obj/include/pdc_prop_pkg.h index db73120ac..52d80efa1 100644 --- a/src/api/pdc_obj/include/pdc_prop_pkg.h +++ b/src/api/pdc_obj/include/pdc_prop_pkg.h @@ -37,9 +37,10 @@ struct _pdc_cont_prop { }; typedef struct pdc_kvtag_t { - char * name; - uint32_t size; - void * value; + char * name; + uint32_t size; + pdc_var_type_t type; + void * value; } pdc_kvtag_t; struct _pdc_transform_state { diff --git a/src/api/pdc_obj/pdc_dt_conv.c b/src/api/pdc_obj/pdc_dt_conv.c index 0806919ee..f18e4aa43 100644 --- a/src/api/pdc_obj/pdc_dt_conv.c +++ b/src/api/pdc_obj/pdc_dt_conv.c @@ -34,10 +34,10 @@ PDC_UNKNOWN = -1, PDC_INT = 0, PDC_FLOAT = 1, PDC_DOUBLE = 2, -PDC_STRING = 3, -PDC_COMPOUND = 4, -PDC_ENUM = 5, -PDC_ARRAY = 6, +PDC_CHAR = 3, +PDC_STRING = 4, +PDC_BOOLEAN = 5, +PDC_SHORT = 6, */ /* Called if overflow is possible */ diff --git a/src/api/pdc_obj/pdc_obj.c b/src/api/pdc_obj/pdc_obj.c index 073ece24c..d402782f9 100644 --- a/src/api/pdc_obj/pdc_obj.c +++ b/src/api/pdc_obj/pdc_obj.c @@ -27,6 +27,7 @@ #include "pdc_malloc.h" #include "pdc_id_pkg.h" #include "pdc_cont.h" +#include "pdc_cont_pkg.h" #include "pdc_prop_pkg.h" #include "pdc_obj_pkg.h" #include "pdc_obj.h" diff --git a/src/api/profiling/CMakeLists.txt b/src/api/profiling/CMakeLists.txt deleted file mode 100644 index 1b2ee8254..000000000 --- a/src/api/profiling/CMakeLists.txt +++ /dev/null @@ -1,120 +0,0 @@ -#------------------------------------------------------------------------------ -# Include source and build directories -#------------------------------------------------------------------------------ -include_directories( - ${PDC_INCLUDES_BUILD_TIME} - ${PROJECT_SOURCE_DIR} - ${PROJECT_BINARY_DIR} - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${PDC_SOURCE_DIR}/src/server/include - ${PDC_SOURCE_DIR}/src/server/pdc_server_region/include - ${PDC_SOURCE_DIR}/src/server/dablooms - ${PDC_SOURCE_DIR}/src/api/include - ${PDC_SOURCE_DIR}/src/api/pdc_obj/include - ${PDC_SOURCE_DIR}/src/api/pdc_region/include - ${PDC_SOURCE_DIR}/src/api/pdc_query/include - ${PDC_SOURCE_DIR}/src/api/pdc_transform/include - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/include - ${PDC_SOURCE_DIR}/src/api/profiling/include - ${PDC_SOURCE_DIR}/src/utils/include - ${MERCURY_INCLUDE_DIR} - ${FASTBIT_INCLUDE_DIR} -) - -install( - FILES - ${CMAKE_BINARY_DIR}/pdc_config.h - DESTINATION - ${PDC_INSTALL_INCLUDE_DIR} - COMPONENT - headers -) - -#------------------------------------------------------------------------------ -# Options -#------------------------------------------------------------------------------ -#add_definitions(-DPDC_ENABLE_MPI=1) -#add_definitions(-DPDC_TIMING=1) -#add_definitions(-DPDC_ENABLE_CHECKPOINT=1) -#add_definitions(-DENABLE_MULTITHREAD=1) - -#------------------------------------------------------------------------------ -# Configure module header files -#------------------------------------------------------------------------------ -# Set unique vars used in the autogenerated config file (symbol import/export) -if(BUILD_SHARED_LIBS) - set(PDC_BUILD_SHARED_LIBS 1) - set(PDC_LIBTYPE SHARED) -else() - set(PDC_BUILD_SHARED_LIBS 0) - set(PDC_LIBTYPE STATIC) -endif() - -#------------------------------------------------------------------------------ -# Set sources -#------------------------------------------------------------------------------ -set(PDC_PROF_SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/pdc_hashtab.c - ${CMAKE_CURRENT_SOURCE_DIR}/pdc_stack_ops.c - ) - -#------------------------------------------------------------------------------ -# Libraries -#------------------------------------------------------------------------------ -# PDCPROF -add_library(pdcprof ${PDC_PROF_SRCS}) -pdc_set_lib_options(pdcprof "pdcprof" ${PDC_LIBTYPE}) - -set(PDC_EXPORTED_LIBS pdcprof ${PDC_EXPORTED_LIBS}) - -#----------------------------------------------------------------------------- -# Specify project header files to be installed -#----------------------------------------------------------------------------- -set(PDC_PROF_HEADERS - ${CMAKE_CURRENT_SOURCE_DIR}/include/pdc_hashtab.h - ${CMAKE_CURRENT_SOURCE_DIR}/include/pdc_stack_ops.h - ) - -#----------------------------------------------------------------------------- -# Add file(s) to CMake Install -#----------------------------------------------------------------------------- -install( - FILES - ${PDC_PROF_HEADERS} - DESTINATION - ${PDC_INSTALL_INCLUDE_DIR} - COMPONENT - headers -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- -install( - TARGETS - pdcprof - EXPORT - ${PDC_EXPORTED_TARGETS} - LIBRARY DESTINATION ${PDC_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${PDC_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${PDC_INSTALL_BIN_DIR} -) - -#------------------------------------------------------------------------------ -# Set variables for parent scope -#------------------------------------------------------------------------------ -# Used by config.cmake.build.in and Testing -set(PDC_INCLUDES_BUILD_TIME - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${PDC_EXT_INCLUDE_DEPENDENCIES} - PARENT_SCOPE -) - -# Used by config.cmake.install.in -set(PDC_INCLUDES_INSTALL_TIME - ${PDC_INSTALL_INCLUDE_DIR} - ${PDC_EXT_INCLUDE_DEPENDENCIES} - PARENT_SCOPE -) diff --git a/src/api/profiling/include/pdc_hashtab.h b/src/api/profiling/include/pdc_hashtab.h deleted file mode 100644 index a664a8ce6..000000000 --- a/src/api/profiling/include/pdc_hashtab.h +++ /dev/null @@ -1,198 +0,0 @@ -/* An expandable hash tables datatype. - Copyright (C) 1999-2017 Free Software Foundation, Inc. - Contributed by Vladimir Makarov (vmakarov@cygnus.com). - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA. */ - -/* This package implements basic hash table functionality. It is possible - to search for an entry, create an entry and destroy an entry. - - Elements in the table are generic pointers. - - The size of the table is not fixed; if the occupancy of the table - grows too high the hash table will be expanded. - - The abstract data implementation is based on generalized Algorithm D - from Knuth's book "The art of computer programming". Hash table is - expanded by creation of new hash table and transferring elements from - the old table to the new table. */ - -#ifndef __HASHTAB_H__ -#define __HASHTAB_H__ - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -/* #include "ansidecl.h" */ -#define PTR void * - -/* The type for a hash code. */ -typedef unsigned int hashval_t; - -/* Callback function pointer types. */ - -/* Calculate hash of a table entry. */ -typedef hashval_t (*htab_hash)(const void *); - -/* Compare a table entry with a possible entry. The entry already in - the table always comes first, so the second element can be of a - different type (but in this case htab_find and htab_find_slot - cannot be used; instead the variants that accept a hash value - must be used). */ -typedef int (*htab_eq)(const void *, const void *); - -/* Cleanup function called whenever a live element is removed from - the hash table. */ -typedef void (*htab_del)(void *); - -/* Function called by htab_traverse for each live element. The first - arg is the slot of the element (which can be passed to htab_clear_slot - if desired), the second arg is the auxiliary pointer handed to - htab_traverse. Return 1 to continue scan, 0 to stop. */ -typedef int (*htab_trav)(void **, void *); - -/* Memory-allocation function, with the same functionality as calloc(). - Iff it returns NULL, the hash table implementation will pass an error - code back to the user, so if your code doesn't handle errors, - best if you use xcalloc instead. */ -typedef void *(*htab_alloc)(size_t, size_t); - -/* We also need a free() routine. */ -typedef void (*htab_free)(void *); - -/* Memory allocation and deallocation; variants which take an extra - argument. */ -typedef void *(*htab_alloc_with_arg)(void *, size_t, size_t); -typedef void (*htab_free_with_arg)(void *, void *); - -/* This macro defines reserved value for empty table entry. */ - -#define HTAB_EMPTY_ENTRY ((PTR)0) - -/* This macro defines reserved value for table entry which contained - a deleted element. */ - -#define HTAB_DELETED_ENTRY ((PTR)1) - -/* Hash tables are of the following type. The structure - (implementation) of this type is not needed for using the hash - tables. All work with hash table should be executed only through - functions mentioned below. The size of this structure is subject to - change. */ - -struct htab { - /* Pointer to hash function. */ - htab_hash hash_f; - - /* Pointer to comparison function. */ - htab_eq eq_f; - - /* Pointer to cleanup function. */ - htab_del del_f; - - /* Table itself. */ - void **entries; - - /* Current size (in entries) of the hash table. */ - size_t size; - - /* Current number of elements including also deleted elements. */ - size_t n_elements; - - /* Current number of deleted elements in the table. */ - size_t n_deleted; - - /* The following member is used for debugging. Its value is number - of all calls of `htab_find_slot' for the hash table. */ - unsigned int searches; - - /* The following member is used for debugging. Its value is number - of collisions fixed for time of work with the hash table. */ - unsigned int collisions; - - /* Pointers to allocate/free functions. */ - htab_alloc alloc_f; - htab_free free_f; - - /* Alternate allocate/free functions, which take an extra argument. */ - void * alloc_arg; - htab_alloc_with_arg alloc_with_arg_f; - htab_free_with_arg free_with_arg_f; - - /* Current size (in entries) of the hash table, as an index into the - table of primes. */ - unsigned int size_prime_index; -}; - -typedef struct htab *htab_t; - -/* An enum saying whether we insert into the hash table or not. */ -enum insert_option { NO_INSERT, INSERT }; - -/* The prototypes of the package functions. */ - -extern htab_t htab_create_alloc(size_t, htab_hash, htab_eq, htab_del, htab_alloc, htab_free); - -extern htab_t htab_create_alloc_ex(size_t, htab_hash, htab_eq, htab_del, void *, htab_alloc_with_arg, - htab_free_with_arg); - -extern htab_t htab_create_typed_alloc(size_t, htab_hash, htab_eq, htab_del, htab_alloc, htab_alloc, - htab_free); - -/* Backward-compatibility functions. */ -extern htab_t htab_create(size_t, htab_hash, htab_eq, htab_del); -extern htab_t htab_try_create(size_t, htab_hash, htab_eq, htab_del); - -extern void htab_set_functions_ex(htab_t, htab_hash, htab_eq, htab_del, void *, htab_alloc_with_arg, - htab_free_with_arg); - -extern void htab_delete(htab_t); -extern void htab_empty(htab_t); - -extern void * htab_find(htab_t, const void *); -extern void **htab_find_slot(htab_t, const void *, enum insert_option); -extern void * htab_find_with_hash(htab_t, const void *, hashval_t); -extern void **htab_find_slot_with_hash(htab_t, const void *, hashval_t, enum insert_option); -extern void htab_clear_slot(htab_t, void **); -extern void htab_remove_elt(htab_t, void *); -extern void htab_remove_elt_with_hash(htab_t, void *, hashval_t); - -extern void htab_traverse(htab_t, htab_trav, void *); -extern void htab_traverse_noresize(htab_t, htab_trav, void *); - -extern size_t htab_size(htab_t); -extern size_t htab_elements(htab_t); -extern double htab_collisions(htab_t); - -/* A hash function for pointers. */ -extern htab_hash htab_hash_pointer; - -/* An equality function for pointers. */ -extern htab_eq htab_eq_pointer; - -/* A hash function for null-terminated strings. */ -extern hashval_t htab_hash_string(const void *); - -/* An iterative hash function for arbitrary data. */ -extern hashval_t iterative_hash(const void *, size_t, hashval_t); -/* Shorthand for hashing something with an intrinsic size. */ -#define iterative_hash_object(OB, INIT) iterative_hash(&OB, sizeof(OB), INIT) - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* __HASHTAB_H */ diff --git a/src/api/profiling/include/pdc_stack_ops.h b/src/api/profiling/include/pdc_stack_ops.h deleted file mode 100644 index 110d03900..000000000 --- a/src/api/profiling/include/pdc_stack_ops.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef _STACK_OPS_H -#define _STACK_OPS_H - -#include "pdc_config.h" -#include "pdc_private.h" -#include -#include -#include -#include -#include - -typedef void *hash_table_t; - -typedef struct profileEntry { - struct profileEntry *next; - struct profileEntry *prev; - const char * ftnkey; - const char * tags; - int64_t count; - int64_t localTotal; - int64_t CumTotal; - int64_t locmin; - int64_t locmax; - double usecTotal; - struct timespec callTime; - struct timespec startTime; - struct timespec totalTime; - struct timespec selfTime; - - struct profileEntry *parent; -} profileEntry_t; - -// typedef enum _boolean {FALSE = 0, TRUE} bool_t; -extern pbool_t enableProfiling; - -#ifndef RESET_TIMER -#define RESET_TIMER(x) (x).tv_sec = (x).tv_nsec = 0; -#endif - -#ifndef TIMER_DIFF -/* t0 = t1 - t2 */ -#define TIMER_DIFF(t0, t1, t2) \ - { \ - if (t2.tv_nsec > (t1).tv_nsec) { \ - (t1).tv_nsec += 1000000000; \ - (t1).tv_sec -= 1; \ - } \ - (t0).tv_sec = (t1).tv_sec - (t2).tv_sec; \ - (t0).tv_nsec = (t1).tv_nsec - (t2).tv_nsec; \ - } -#endif - -#ifndef TIMER_ADD -/* t0 += t1 */ -#define TIMER_ADD(t0, t1) \ - { \ - (t0).tv_sec += (t1).tv_sec; \ - if (((t0).tv_nsec += (t1).tv_nsec) > 10000000000) { \ - (t0).tv_sec += 1; \ - (t0).tv_nsec -= 10000000000; \ - } \ - } -#endif - -void initialize_profile(void **table, size_t tabsize); -void finalize_profile(); -void push(const char *ftnkey, const char *tags); -void pop(); - -#endif diff --git a/src/api/profiling/pdc_hashtab.c b/src/api/profiling/pdc_hashtab.c deleted file mode 100644 index e59b7e3ba..000000000 --- a/src/api/profiling/pdc_hashtab.c +++ /dev/null @@ -1,540 +0,0 @@ -/* An expandable hash tables datatype. - Copyright (C) 1999, 2000, 2001, 2002 Free Software Foundation, Inc. - Contributed by Vladimir Makarov (vmakarov@cygnus.com). - -This file is part of the libiberty library. -Libiberty is free software; you can redistribute it and/or -modify it under the terms of the GNU Library General Public -License as published by the Free Software Foundation; either -version 2 of the License, or (at your option) any later version. - -Libiberty is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Library General Public License for more details. - -You should have received a copy of the GNU Library General Public -License along with libiberty; see the file COPYING.LIB. If -not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, -Boston, MA 02111-1307, USA. */ - -/* This package implements basic hash table functionality. It is possible - to search for an entry, create an entry and destroy an entry. - - Elements in the table are generic pointers. - - The size of the table is not fixed; if the occupancy of the table - grows too high the hash table will be expanded. - - The abstract data implementation is based on generalized Algorithm D - from Knuth's book "The art of computer programming". Hash table is - expanded by creation of new hash table and transferring elements from - the old table to the new table. */ - -#include -#include -#include -#include -#include "pdc_config.h" -#include "pdc_hashtab.h" - -/* This macro defines reserved value for empty table entry. */ - -#define EMPTY_ENTRY ((PTR)0) - -/* This macro defines reserved value for table entry which contained - a deleted element. */ - -#define DELETED_ENTRY ((PTR)1) - -static unsigned long higher_prime_number(unsigned long); -static hashval_t hash_pointer(const void *); -static int eq_pointer(const void *, const void *); -static int htab_expand(htab_t); -static PTR * find_empty_slot_for_expand(htab_t, hashval_t); - -/* At some point, we could make these be NULL, and modify the - hash-table routines to handle NULL specially; that would avoid - function-call overhead for the common case of hashing pointers. */ -htab_hash htab_hash_pointer = hash_pointer; -htab_eq htab_eq_pointer = eq_pointer; - -/* The following function returns a nearest prime number which is - greater than N, and near a power of two. */ - -static unsigned long higher_prime_number(n) unsigned long n; -{ - /* These are primes that are near, but slightly smaller than, a - power of two. */ - static const unsigned long primes[] = { - (unsigned long)7, - (unsigned long)13, - (unsigned long)31, - (unsigned long)61, - (unsigned long)127, - (unsigned long)251, - (unsigned long)509, - (unsigned long)1021, - (unsigned long)2039, - (unsigned long)4093, - (unsigned long)8191, - (unsigned long)16381, - (unsigned long)32749, - (unsigned long)65521, - (unsigned long)131071, - (unsigned long)262139, - (unsigned long)524287, - (unsigned long)1048573, - (unsigned long)2097143, - (unsigned long)4194301, - (unsigned long)8388593, - (unsigned long)16777213, - (unsigned long)33554393, - (unsigned long)67108859, - (unsigned long)134217689, - (unsigned long)268435399, - (unsigned long)536870909, - (unsigned long)1073741789, - (unsigned long)2147483647, - /* 4294967291L */ - ((unsigned long)2147483647) + ((unsigned long)2147483644), - }; - - const unsigned long *low = &primes[0]; - const unsigned long *high = &primes[sizeof(primes) / sizeof(primes[0])]; - - while (low != high) { - const unsigned long *mid = low + (high - low) / 2; - if (n > *mid) - low = mid + 1; - else - high = mid; - } - - /* If we've run out of primes, abort. */ - if (n > *low) { - fprintf(stderr, "Cannot find prime bigger than %lu\n", n); - abort(); - } - - return *low; -} - -/* Returns a hash code for P. */ - -static hashval_t hash_pointer(p) const PTR p; -{ - return (hashval_t)((long)p >> 3); -} - -/* Returns non-zero if P1 and P2 are equal. */ - -static int eq_pointer(p1, p2) const PTR p1; -const PTR p2; -{ - return p1 == p2; -} - -/* This function creates table with length slightly longer than given - source length. Created hash table is initiated as empty (all the - hash table entries are EMPTY_ENTRY). The function returns the - created hash table, or NULL if memory allocation fails. */ - -htab_t htab_create_alloc(size, hash_f, eq_f, del_f, alloc_f, free_f) size_t size; -htab_hash hash_f; -htab_eq eq_f; -htab_del del_f; -htab_alloc alloc_f; -htab_free free_f; -{ - htab_t result; - size = higher_prime_number(size); - result = (htab_t)(*alloc_f)(1, sizeof(struct htab)); - if (result == NULL) - return NULL; - - result->entries = (PTR *)(*alloc_f)(size, sizeof(PTR)); - if (result->entries == NULL) { - if (free_f != NULL) - (*free_f)(result); - return NULL; - } - result->size = size; - result->hash_f = hash_f; - result->eq_f = eq_f; - result->del_f = del_f; - result->alloc_f = alloc_f; - result->free_f = free_f; - - return result; -} - -/* These functions exist solely for backward compatibility. */ - -#undef htab_create -htab_t htab_create(size, hash_f, eq_f, del_f) size_t size; -htab_hash hash_f; -htab_eq eq_f; -htab_del del_f; -{ - return htab_create_alloc(size, hash_f, eq_f, del_f, calloc, free); -} - -htab_t htab_try_create(size, hash_f, eq_f, del_f) size_t size; -htab_hash hash_f; -htab_eq eq_f; -htab_del del_f; -{ - return htab_create_alloc(size, hash_f, eq_f, del_f, calloc, free); -} - -/* This function frees all memory allocated for given hash table. - Naturally the hash table must already exist. */ - -void htab_delete(htab) htab_t htab; -{ - int i; - - if (htab->del_f) { - for (i = htab->size - 1; i >= 0; i--) - if (htab->entries[i] != EMPTY_ENTRY && htab->entries[i] != DELETED_ENTRY) - (*htab->del_f)(htab->entries[i]); - } - if (htab->free_f != NULL) { - (*htab->free_f)(htab->entries); - (*htab->free_f)(htab); - } -} - -/* This function clears all entries in the given hash table. */ - -void htab_empty(htab) htab_t htab; -{ - int i; - - if (htab->del_f) - for (i = htab->size - 1; i >= 0; i--) - if (htab->entries[i] != EMPTY_ENTRY && htab->entries[i] != DELETED_ENTRY) - (*htab->del_f)(htab->entries[i]); - - memset(htab->entries, 0, htab->size * sizeof(PTR)); -} - -/* Similar to htab_find_slot, but without several unwanted side effects: - - Does not call htab->eq_f when it finds an existing entry. - - Does not change the count of elements/searches/collisions in the - hash table. - This function also assumes there are no deleted entries in the table. - HASH is the hash value for the element to be inserted. */ - -static PTR *find_empty_slot_for_expand(htab, hash) htab_t htab; -hashval_t hash; -{ - size_t size = htab->size; - unsigned int index = hash % size; - PTR * slot = htab->entries + index; - hashval_t hash2; - - if (*slot == EMPTY_ENTRY) - return slot; - else if (*slot == DELETED_ENTRY) - abort(); - - hash2 = 1 + hash % (size - 2); - for (;;) { - index += hash2; - if (index >= size) - index -= size; - - slot = htab->entries + index; - if (*slot == EMPTY_ENTRY) - return slot; - else if (*slot == DELETED_ENTRY) - abort(); - } -} - -/* The following function changes size of memory allocated for the - entries and repeatedly inserts the table elements. The occupancy - of the table after the call will be about 50%. Naturally the hash - table must already exist. Remember also that the place of the - table entries is changed. If memory allocation failures are allowed, - this function will return zero, indicating that the table could not be - expanded. If all goes well, it will return a non-zero value. */ - -static int htab_expand(htab) htab_t htab; -{ - PTR *oentries; - PTR *olimit; - PTR *p; - PTR *nentries; - - oentries = htab->entries; - olimit = oentries + htab->size; - - htab->size = higher_prime_number(htab->size * 2); - - nentries = (PTR *)(*htab->alloc_f)(htab->size, sizeof(PTR *)); - if (nentries == NULL) - return 0; - htab->entries = nentries; - htab->n_elements -= htab->n_deleted; - htab->n_deleted = 0; - - p = oentries; - do { - PTR x = *p; - - if (x != EMPTY_ENTRY && x != DELETED_ENTRY) { - PTR *q = find_empty_slot_for_expand(htab, (*htab->hash_f)(x)); - *q = x; - } - p++; - } while (p < olimit); - - if (htab->free_f != NULL) - (*htab->free_f)(oentries); - - return 1; -} - -/* This function searches for a hash table entry equal to the given - element. It cannot be used to insert or delete an element. */ - -PTR htab_find_with_hash(htab, element, hash) htab_t htab; -const PTR element; -hashval_t hash; -{ - unsigned int index; - hashval_t hash2; - size_t size; - PTR entry; - - htab->searches++; - size = htab->size; - index = hash % size; - - entry = htab->entries[index]; - if (entry == EMPTY_ENTRY || (entry != DELETED_ENTRY && (*htab->eq_f)(entry, element))) - return entry; - - hash2 = 1 + hash % (size - 2); - - for (;;) { - htab->collisions++; - index += hash2; - if (index >= size) - index -= size; - - entry = htab->entries[index]; - if (entry == EMPTY_ENTRY || (entry != DELETED_ENTRY && (*htab->eq_f)(entry, element))) - return entry; - } -} - -/* Like htab_find_slot_with_hash, but compute the hash value from the - element. */ - -PTR htab_find(htab, element) htab_t htab; -const PTR element; -{ - return htab_find_with_hash(htab, element, (*htab->hash_f)(element)); -} - -/* This function searches for a hash table slot containing an entry - equal to the given element. To delete an entry, call this with - INSERT = 0, then call htab_clear_slot on the slot returned (possibly - after doing some checks). To insert an entry, call this with - INSERT = 1, then write the value you want into the returned slot. - When inserting an entry, NULL may be returned if memory allocation - fails. */ - -PTR * htab_find_slot_with_hash(htab, element, hash, insert) htab_t htab; -const PTR element; -hashval_t hash; -enum insert_option insert; -{ - PTR * first_deleted_slot; - unsigned int index; - hashval_t hash2; - size_t size; - PTR entry; - - if (insert == INSERT && htab->size * 3 <= htab->n_elements * 4 && htab_expand(htab) == 0) - return NULL; - - size = htab->size; - index = hash % size; - - htab->searches++; - first_deleted_slot = NULL; - - entry = htab->entries[index]; - if (entry == EMPTY_ENTRY) - goto empty_entry; - else if (entry == DELETED_ENTRY) - first_deleted_slot = &htab->entries[index]; - else if ((*htab->eq_f)(entry, element)) - return &htab->entries[index]; - - hash2 = 1 + hash % (size - 2); - for (;;) { - htab->collisions++; - index += hash2; - if (index >= size) - index -= size; - - entry = htab->entries[index]; - if (entry == EMPTY_ENTRY) - goto empty_entry; - else if (entry == DELETED_ENTRY) { - if (!first_deleted_slot) - first_deleted_slot = &htab->entries[index]; - } - else if ((*htab->eq_f)(entry, element)) - return &htab->entries[index]; - } - -empty_entry: - if (insert == NO_INSERT) - return NULL; - - htab->n_elements++; - - if (first_deleted_slot) { - *first_deleted_slot = EMPTY_ENTRY; - return first_deleted_slot; - } - - return &htab->entries[index]; -} - -/* Like htab_find_slot_with_hash, but compute the hash value from the - element. */ - -PTR * htab_find_slot(htab, element, insert) htab_t htab; -const PTR element; -enum insert_option insert; -{ - return htab_find_slot_with_hash(htab, element, (*htab->hash_f)(element), insert); -} - -/* This function deletes an element with the given value from hash - table. If there is no matching element in the hash table, this - function does nothing. */ - -void htab_remove_elt(htab, element) htab_t htab; -PTR element; -{ - PTR *slot; - - slot = htab_find_slot(htab, element, NO_INSERT); - if (*slot == EMPTY_ENTRY) - return; - - if (htab->del_f) - (*htab->del_f)(*slot); - - *slot = DELETED_ENTRY; - htab->n_deleted++; -} - -/* This function clears a specified slot in a hash table. It is - useful when you've already done the lookup and don't want to do it - again. */ - -void htab_clear_slot(htab, slot) htab_t htab; -PTR *slot; -{ - if (slot < htab->entries || slot >= htab->entries + htab->size || *slot == EMPTY_ENTRY || - *slot == DELETED_ENTRY) - abort(); - - if (htab->del_f) - (*htab->del_f)(*slot); - - *slot = DELETED_ENTRY; - htab->n_deleted++; -} - -/* This function scans over the entire hash table calling - CALLBACK for each live entry. If CALLBACK returns false, - the iteration stops. INFO is passed as CALLBACK's second - argument. */ - -void htab_traverse(htab, callback, info) htab_t htab; -htab_trav callback; -PTR info; -{ - PTR *slot = htab->entries; - PTR *limit = slot + htab->size; - - do { - PTR x = *slot; - if (x != EMPTY_ENTRY && x != DELETED_ENTRY) - if (!(*callback)(slot, info)) - break; - } while (++slot < limit); -} - -/* Return the current size of given hash table. */ - -size_t htab_size(htab) htab_t htab; -{ - return htab->size; -} - -/* Return the current number of elements in given hash table. */ - -size_t htab_elements(htab) htab_t htab; -{ - return htab->n_elements - htab->n_deleted; -} - -/* Return the fraction of fixed collisions during all work with given - hash table. */ - -double htab_collisions(htab) htab_t htab; -{ - if (htab->searches == 0) - return 0.0; - - return (double)htab->collisions / (double)htab->searches; -} - -/* Hash P as a null-terminated string. - - Copied from gcc/hashtable.c. Zack had the following to say with respect - to applicability, though note that unlike hashtable.c, this hash table - implementation re-hashes rather than chain buckets. - - http://gcc.gnu.org/ml/gcc-patches/2001-08/msg01021.html - From: Zack Weinberg - Date: Fri, 17 Aug 2001 02:15:56 -0400 - - I got it by extracting all the identifiers from all the source code - I had lying around in mid-1999, and testing many recurrences of - the form "H_n = H_{n-1} * K + c_n * L + M" where K, L, M were either - prime numbers or the appropriate identity. This was the best one. - I don't remember exactly what constituted "best", except I was - looking at bucket-length distributions mostly. - - So it should be very good at hashing identifiers, but might not be - as good at arbitrary strings. - - I'll add that it thoroughly trounces the hash functions recommended - for this use at http://burtleburtle.net/bob/hash/index.html, both - on speed and bucket distribution. I haven't tried it against the - function they just started using for Perl's hashes. */ - -hashval_t htab_hash_string(p) const PTR p; -{ - const unsigned char *str = (const unsigned char *)p; - hashval_t r = 0; - unsigned char c; - - while ((c = *str++) != 0) - r = r * 67 + c - 113; - - return r; -} diff --git a/src/api/profiling/pdc_stack_ops.c b/src/api/profiling/pdc_stack_ops.c deleted file mode 100644 index fab9f274a..000000000 --- a/src/api/profiling/pdc_stack_ops.c +++ /dev/null @@ -1,264 +0,0 @@ -#include -#include -#include -#include "pdc_stack_ops.h" -#include "pdc_hashtab.h" - -profileEntry_t *calltree = NULL; -profileEntry_t *freelist = NULL; - -static int profilerrors = 0; - -hash_table_t hashtable; - -htab_t thisHashTable; - -/* For now we disable profiling (by default) - * Note that one can always ENABLE it by set the - * environment variable "PROFILE_ENABLE=true" - */ -pbool_t enableProfiling = FALSE; - -/* - * The idea of this implementation is to simulate the call stack - * of the running application. Each function that we care about - * begins with a FUNC_ENTER(x) declaration and finishes with - * FUNC_LEAVE(ret). These of course are macros and under - * the condition that we enable profiling, these expand into - * push and pop operations which we define below. - * - * Example: suppose that a user application is defined as follows - * int main() { - * a(); - * b(); - * c(); - * return 0; - * } - * - * void a() { - * aa(); - * aaa(); - * ab(); - * } - * - * void b() { - * bb(); - * bbb(); - * bc(); - * } - * - * void c() { - * cc(); - * ccc(); - * ca(); - * } - * - * Assume that all of the internal functions only make system calls - * or 3rd party libraries, i.e. the underlying functions will NOT - * be profiled. - * - * The implementation of stack_ops will maintain a call tree - * that mirrors that of the actual program, i.e. the alltree data - * structure will contain something like the following as we enter - * the first function contained by a(): - * - * ("main") --> ("a") --> ("aa") - * - * The entry for "main" has a /start_time and no /total_time - * Similarly, "a" has it's own /start_time and no /total_time - * The final entry: "aa" has a start-time and just prior to - * the return to it's parent ("a"), we sample the real-time - * clock as part of the POP functionality. Using the current - * time minus the start-time we establish the raw total elapsed - * time for the current function. - * NOTE: The actual runtime spent within the function is - * a calculation which subtracts out the total elapsed times - * of all of the lower-level functions, e.g. suppose ("a") - * has a total runtime of 10. If the total runtime of ("aa") - * in the simple call chain shown above is 5, then the actual - * profiled time spent in ("a") is 10 - 5 = 5. - * Ultimately, if were to execute the entire program and then - * sum all of the individual profile times, the total should - * match the execution time of the program. - */ - -void -push(const char *ftnkey, const char *tags) -{ - profileEntry_t *thisEntry; - if (freelist != NULL) { - thisEntry = freelist; - freelist = thisEntry->next; - } - else { - if ((thisEntry = (profileEntry_t *)malloc(sizeof(profileEntry_t))) == NULL) { - perror("malloc"); - profilerrors++; - } - } - - if (profilerrors) - return; - thisEntry->ftnkey = ftnkey; - thisEntry->tags = tags; - thisEntry->prev = calltree; - thisEntry->next = NULL; - calltree = thisEntry; - - /* Timing */ - clock_gettime(CLOCK_REALTIME, &thisEntry->startTime); - RESET_TIMER(thisEntry->callTime); - return; -} - -void -pop() -{ - struct timespec current_time; - profileEntry_t *master; - profileEntry_t *thisEntry = calltree; - int update_entry = TRUE; - if (thisEntry == NULL) - return; /* This shouldn't happen */ - - /* Timing */ - clock_gettime(CLOCK_REALTIME, ¤t_time); - TIMER_DIFF(thisEntry->totalTime, current_time, thisEntry->startTime); - TIMER_DIFF(thisEntry->selfTime, thisEntry->totalTime, thisEntry->callTime); - calltree = thisEntry->prev; - if (calltree != NULL) { - TIMER_ADD(calltree->callTime, thisEntry->totalTime); - } - /* Check to see if this function has already been added to the hashtable */ - void **tableEntry = htab_find_slot(thisHashTable, thisEntry, INSERT); - if (*tableEntry == NULL) { - /* No table entry found so add it now ... */ - master = (profileEntry_t *)malloc(sizeof(profileEntry_t)); - if (master) { - thisEntry->count = 1; - memcpy(master, thisEntry, sizeof(profileEntry_t)); - *tableEntry = master; - } - update_entry = FALSE; - } - - if (update_entry) { - master = *(profileEntry_t **)tableEntry; - master->count++; - TIMER_ADD(master->totalTime, thisEntry->totalTime); - TIMER_ADD(master->selfTime, thisEntry->selfTime); - } - - /* Rather than freeing the container, we add the - * current entry onto the freelist. - */ - thisEntry->next = freelist; - freelist = thisEntry; -} - -hashval_t -hash_profile_entry(const void *p) -{ - const profileEntry_t *thisEntry = (const profileEntry_t *)p; - return htab_hash_string(thisEntry->ftnkey); -} - -int -eq_profile_entry(const void *a, const void *b) -{ - const profileEntry_t *tp_a = (const profileEntry_t *)a; - const profileEntry_t *tp_b = (const profileEntry_t *)b; - return (tp_a->ftnkey == tp_b->ftnkey); -} - -void -initialize_profile(void **hashtab, size_t size) -{ - if (*hashtab == NULL) { - if ((thisHashTable = htab_try_create(size, hash_profile_entry, eq_profile_entry, free)) == NULL) { - return; - } - *hashtab = thisHashTable; - } -} - -int -show_profile_info(void **ht_live_entry, void *extraInfo ATTRIBUTE(unused)) -{ - static int count = 0; - char * LineBreak = "------------------------------------------------------------------------------"; - char * header = " item calls Time/call [Sec,nSec]\tftn_name"; - const profileEntry_t *thisEntry = *(const profileEntry_t **)ht_live_entry; - - if (thisEntry) { - struct timespec totalTime; - int64_t totalCalls = thisEntry->count; - if (count == 0) - puts(header); - totalTime = thisEntry->totalTime; - printf("%s\n %d\t%-6" PRId64 " %6" PRId64 ",%6" PRId64 "\t\t %s\n", LineBreak, ++count, totalCalls, - totalTime.tv_sec / totalCalls, totalTime.tv_nsec / totalCalls, thisEntry->ftnkey); - } - - return TRUE; -} - -/* Returns 1 if we set enableProfiling to TRUE - * otherwise returns 0. - */ -int -toggle_profile_enable() -{ - if (enableProfiling == FALSE) - enableProfiling = TRUE; - else - enableProfiling = FALSE; - - return (enableProfiling ? 1 : 0); -} - -/* These functions should be used when we've actually built the profiler as a shared library. - * Note: One might check an environment variable to see if a non-default size - * for the hashtable initialization should be used... - * The profile_fini should probably be used to dump the contents of the profile - * hashtable. - */ - -void __attribute__((constructor)) profile_init(void) -{ - int default_HashtableSize = 128; - char *size_override = NULL; - char *profile_enable = getenv("PROFILE_ENABLE"); - if (profile_enable != NULL) { - if (strcasecmp(profile_enable, "true") == 0) { - enableProfiling = TRUE; - } - else if (strcasecmp(profile_enable, "false") == 0) { - enableProfiling = FALSE; - } - } - // While it is tempting to skip creating a hashtable - // if we've disabled profiling (see above), I want - // to give the user the ability at runtime to - // possibly enable everything... - // I don't currently include any APIs to enable - // or disable profiling at runtime, but that is - // on the TODO list. - - size_override = getenv("PROFILE_HASHTABLESIZE"); - if (size_override != NULL) { - int override_value = atoi(size_override); - if (override_value > 0) { - default_HashtableSize = override_value; - } - } - initialize_profile(&hashtable, default_HashtableSize); -} - -void __attribute__((destructor)) finalize_profile(void) -{ - int count = 1; - if (thisHashTable != NULL) { - htab_traverse(thisHashTable, show_profile_info, &count); - } -} diff --git a/src/commons/CMakeLists.txt b/src/commons/CMakeLists.txt new file mode 100644 index 000000000..f29c25a7c --- /dev/null +++ b/src/commons/CMakeLists.txt @@ -0,0 +1,227 @@ +#------------------------------------------------------------------------------ +# PDC Commons +#------------------------------------------------------------------------------ + +set(PDC_COMMON_LIBRARY_NAME pdc_commons CACHE INTERNAL "") + +#------------------------------------------------------------------------------ +# External dependencies +#------------------------------------------------------------------------------ + +# #set(PDC_EXT_LIB_DEPENDENCIES ${PDC_COMMON_LIBRARY_NAME} ${PDC_EXT_LIB_DEPENDENCIES}) +# set(PDC_EXT_INCLUDE_DEPENDENCIES ${CMAKE_CURRENT_SOURCE_DIR}/profiling) +# set(PDC_EXPORTED_LIBS pdcprof) + +if(THREADS_HAVE_PTHREAD_ARG) + set_property(TARGET ${PDC_COMMON_LIBRARY_NAME} PROPERTY COMPILE_OPTIONS "-pthread") + set_property(TARGET ${PDC_COMMON_LIBRARY_NAME} PROPERTY INTERFACE_COMPILE_OPTIONS "-pthread") +endif() + + +# Mercury +find_package(MERCURY REQUIRED) +if(MERCURY_FOUND) + message(STATUS "mercury dir ${MERCURY_DIR}") + if(DEFINED MERCURY_DIR AND NOT "${MERCURY_DIR}" STREQUAL "") + # If MERCURY_DIR contains "share/" (or "share" is at the end), truncate it along with everything following it + string(REGEX REPLACE "/share.*" "" trimmed_mercury_dir ${MERCURY_DIR}) + # Check if the trimmed_mercury_dir ends with "/mercury" + string(REGEX MATCH ".*/mercury$" is_mercury_home ${trimmed_mercury_dir}) + # If trimmed_mercury_dir ends with "/mercury", consider it as the home directory + if(is_mercury_home) + set(MERCURY_HOME ${trimmed_mercury_dir}) + else() + # If not ending with "mercury", it's likely an error + message(FATAL_ERROR "Cannot determine MERCURY_HOME from MERCURY_DIR.") + endif() + + message("Mercury home is set to ${MERCURY_HOME}") + + set(MERCURY_INCLUDE_DIR ${MERCURY_HOME}/include) + set(MERCURY_LIBRARY_DIR ${MERCURY_HOME}/lib) + endif() + message(STATUS "mercury include dir ${MERCURY_INCLUDE_DIR}") + message(STATUS "mercury lib dir ${MERCURY_LIBRARY_DIR}") + set(PDC_EXT_INCLUDE_DEPENDENCIES ${MERCURY_INCLUDE_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + ) + set(PDC_EXT_LIB_DEPENDENCIES mercury ${PDC_EXT_LIB_DEPENDENCIES}) +endif() + +include_directories(${PDC_EXT_INCLUDE_DEPENDENCIES}) + +#------------------------------------------------------------------------------ +# Include directories +#------------------------------------------------------------------------------ + +# Get a list of all directories that contain header files +file(GLOB_RECURSE LOCAL_INCLUDE_DIRS "*.h") + +# Remove the /filename.h at the end of each directory +list(TRANSFORM LOCAL_INCLUDE_DIRS REPLACE "/[^/]*$" "") + +# Remove duplicates +list(REMOVE_DUPLICATES LOCAL_INCLUDE_DIRS) + +set(PDC_COMMONS_INCLUDE_DIRS + ${LOCAL_INCLUDE_DIRS} + ${PDC_INCLUDES_BUILD_TIME} + ${PROJECT_BINARY_DIR} + ${PDC_SOURCE_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} +) + +include_directories( + ${PDC_COMMONS_INCLUDE_DIRS} +) + +message(STATUS "PDC_COMMONS_INCLUDE_DIRS: ${PDC_COMMONS_INCLUDE_DIRS}") + +install( + FILES + ${CMAKE_BINARY_DIR}/pdc_config.h + DESTINATION + ${PDC_INSTALL_INCLUDE_DIR} + COMPONENT + headers +) + +#------------------------------------------------------------------------------ +# Configure module header files +#------------------------------------------------------------------------------ +# Set unique vars used in the autogenerated config file (symbol import/export) +if(BUILD_SHARED_LIBS) + set(PDC_BUILD_SHARED_LIBS 1) + set(PDC_LIBTYPE SHARED) +else() + set(PDC_BUILD_SHARED_LIBS 0) + set(PDC_LIBTYPE STATIC) +endif() + +if(PDC_ENABLE_TIMING) + add_definitions(-DPDC_TIMING=1) +endif() + +#------------------------------------------------------------------------------ +# Set sources +#------------------------------------------------------------------------------ + +# Collect all source files +file(GLOB_RECURSE PDC_COMMONS_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.c) +file(GLOB_RECURSE PDC_COMMONS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.h) + + +#------------------------------------------------------------------------------ +# Libraries +#------------------------------------------------------------------------------ +# PDC COMMONS + + +add_library(${PDC_COMMON_LIBRARY_NAME} ${PDC_LIBTYPE} ${PDC_COMMONS_SOURCES} ${PDC_COMMONS_HEADERS}) + +target_include_directories(${PDC_COMMON_LIBRARY_NAME} + PUBLIC "$" + $ +) + +target_link_libraries(${PDC_COMMON_LIBRARY_NAME} INTERFACE + ${PDC_EXT_LIB_DEPENDENCIES} +) + +set(PDC_EXPORTED_LIBS ${PDC_COMMON_LIBRARY_NAME} ${PDC_EXPORTED_LIBS}) + +#----------------------------------------------------------------------------- +# Specify project header files to be installed +#----------------------------------------------------------------------------- + + +set(PDC_PUBLIC_HEADERS "") +set(PUBLIC_HEADER_DIR_LIST + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/generic/include + ${CMAKE_CURRENT_SOURCE_DIR}/profiling/include + ${CMAKE_CURRENT_SOURCE_DIR}/utils/include +) + +foreach(_header_dir ${PUBLIC_HEADER_DIR_LIST}) + file(GLOB_RECURSE _dir_headers ${_header_dir}/*.h) + list(APPEND PDC_PUBLIC_HEADERS ${_dir_headers}) +endforeach() + +set(PDC_COMMONS_HEADERS + ${PDC_PUBLIC_HEADERS} + ${PROJECT_BINARY_DIR}/pdc_config_sys.h + ${PROJECT_BINARY_DIR}/pdc_config.h + ) + +#----------------------------------------------------------------------------- +# Add file(s) to CMake Install +#----------------------------------------------------------------------------- +install( + FILES + ${PDC_COMMONS_HEADERS} + DESTINATION + ${PDC_INSTALL_INCLUDE_DIR} + COMPONENT + headers +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install +#----------------------------------------------------------------------------- + +install( + TARGETS + ${PDC_COMMON_LIBRARY_NAME} + EXPORT + ${PDC_EXPORTED_TARGETS} + LIBRARY DESTINATION ${PDC_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${PDC_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${PDC_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${PDC_EXPORTED_TARGETS} + DESTINATION + ${PDC_INSTALL_DATA_DIR}/cmake/pdc + FILE + ${PDC_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +if(NOT PDC_EXTERNALLY_CONFIGURED) +EXPORT ( + TARGETS + ${PDC_EXPORTED_LIBS} + FILE + ${PDC_EXPORTED_TARGETS}.cmake +) +endif() + +#------------------------------------------------------------------------------ +# Set variables for parent scope +#------------------------------------------------------------------------------ +# Used by config.cmake.build.in and Testing +set(PDC_INCLUDES_BUILD_TIME + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + ${PDC_COMMONS_INCLUDE_DIRS} + PARENT_SCOPE +) + +# Used by config.cmake.install.in +set(PDC_INCLUDES_INSTALL_TIME + ${PDC_COMMONS_INCLUDE_DIRS} + ${PDC_INSTALL_INCLUDE_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + PARENT_SCOPE +) + +set(PDC_COMMONS_LIBRARIES ${PDC_COMMON_LIBRARY_NAME} PARENT_SCOPE) \ No newline at end of file diff --git a/src/commons/generic/include/pdc_generic.h b/src/commons/generic/include/pdc_generic.h new file mode 100644 index 000000000..47322b553 --- /dev/null +++ b/src/commons/generic/include/pdc_generic.h @@ -0,0 +1,219 @@ +#ifndef PDC_GENERIC_H +#define PDC_GENERIC_H + +#include +#include +#include +#include +#include + +#ifndef __cplusplus +#if __STDC_VERSION__ >= 199901L +/* C99 or later */ +#include +#else +/* Pre-C99 */ +typedef enum { false = 0, true = 1 } bool; +#endif +#endif + +typedef enum { + PDC_UNKNOWN = -1, /* error */ + PDC_INT = 0, /* integer types (identical to int32_t) */ + PDC_FLOAT = 1, /* floating-point types */ + PDC_DOUBLE = 2, /* double types */ + PDC_CHAR = 3, /* character types */ + PDC_STRING = 4, /* string types */ + PDC_BOOLEAN = 5, /* boolean types */ + PDC_SHORT = 6, /* short types */ + PDC_UINT = 7, /* unsigned integer types (identical to uint32_t) */ + PDC_INT64 = 8, /* 64-bit integer types */ + PDC_UINT64 = 9, /* 64-bit unsigned integer types */ + PDC_INT16 = 10, /* 16-bit integer types */ + PDC_INT8 = 11, /* 8-bit integer types */ + PDC_UINT8 = 12, /* 8-bit unsigned integer types */ + PDC_UINT16 = 13, /* 16-bit unsigned integer types */ + PDC_INT32 = 14, /* 32-bit integer types */ + PDC_UINT32 = 15, /* 32-bit unsigned integer types */ + PDC_LONG = 16, /* long types */ + PDC_VOID_PTR = 17, /* void pointer type */ + PDC_SIZE_T = 18, /* size_t type */ + PDC_TYPE_COUNT = 19 /* this is the number of var types and has to be the last */ +} pdc_c_var_type_t; + +typedef pdc_c_var_type_t PDC_CType; + +typedef enum { + PDC_CLS_SCALAR, + PDC_CLS_ARRAY, + PDC_CLS_ENUM, // not implemented, users can use PDC_CT_INT + PDC_CLS_STRUCT, // not implemented, users can use embedded key value pairs for the members in a struct + PDC_CLS_UNION, // not implemented, users can use embedded key value pairs for the only one member value + // in a union. + PDC_CLS_POINTER, // not implemented, users can use PDC_CT_INT64_T to store the pointer address, but + // won't work for distributed memory. + PDC_CLS_FUNCTION, // not implemented, users can use PDC_CT_INT64_T to store the function address, but + // won't work for distributed memory. + PDC_CLS_COUNT // just the count of the enum. +} pdc_c_var_class_t; + +typedef pdc_c_var_class_t PDC_CType_Class; + +// clang-format off +static const size_t DataTypeSizes[PDC_TYPE_COUNT] = { + sizeof(int), + sizeof(float), + sizeof(double), + sizeof(char), + sizeof(char *), + sizeof(bool), + sizeof(short), + sizeof(unsigned int), + sizeof(int64_t), + sizeof(uint64_t), + sizeof(int16_t), + sizeof(int8_t), + sizeof(uint8_t), + sizeof(uint16_t), + sizeof(int32_t), + sizeof(uint32_t), + sizeof(long), + sizeof(void *), + sizeof(size_t) +}; + +static const char *DataTypeNames[PDC_TYPE_COUNT] = { + "int", + "float", + "double", + "char", + "char*", + "bool", + "short", + "unsigned int", + "int64_t", + "uint64_t", + "int16_t", + "int8_t", + "uint8_t", + "uint16_t", + "int32_t", + "uint32_t", + "long", + "void*", + "size_t" +}; + +static const char *DataTypeEnumNames[PDC_TYPE_COUNT] = { + "PDC_INT", + "PDC_FLOAT", + "PDC_DOUBLE", + "PDC_CHAR", + "PDC_STRING", + "PDC_BOOLEAN", + "PDC_SHORT", + "PDC_UINT", + "PDC_INT64", + "PDC_UINT64", + "PDC_INT16", + "PDC_INT8", + "PDC_UINT8", + "PDC_UINT16", + "PDC_INT32", + "PDC_UINT32", + "PDC_LONG", + "PDC_VOID_PTR", + "PDC_SIZE_T" +}; + +static const char *DataTypeFormat[PDC_TYPE_COUNT] = { + "%d", // int + "%f", // float + "%lf", // double + "%c", // char + "%s", // char* + "%d", // bool (represented as an integer) + "%hd", // short + "%u", // unsigned int + "%lld", // int64_t + "%llu", // uint64_t + "%hd", // int16_t + "%hhd", // int8_t + "%hhu", // uint8_t + "%hu", // uint16_t + "%d", // int32_t + "%u", // uint32_t + "%ld", // long + "%p", // void* (pointer) + "%zu" // size_t +}; + +// clang-format on + +static const char * +get_enum_name_by_dtype(pdc_c_var_type_t type) +{ + if (type < 0 || type >= PDC_TYPE_COUNT) { + return NULL; + } + return DataTypeEnumNames[type]; +} + +static const size_t +get_size_by_dtype(pdc_c_var_type_t type) +{ + if (type < 0 || type >= PDC_TYPE_COUNT) { + return 0; + } + return DataTypeSizes[type]; +} + +static const size_t +get_size_by_class_n_type(void *data, size_t item_count, pdc_c_var_class_t pdc_class, + pdc_c_var_type_t pdc_type) +{ + size_t size = 0; + if (pdc_class == PDC_CLS_SCALAR) { + if (pdc_type == PDC_STRING) { + size = (strlen((char *)data) + 1) * sizeof(char); + } + else { + size = get_size_by_dtype(pdc_type); + } + } + else if (pdc_class == PDC_CLS_ARRAY) { + if (pdc_type == PDC_STRING) { + char **str_arr = (char **)data; + int i = 0; + for (i = 0; i < item_count; i++) { + size = size + (strlen(str_arr[i]) + 1) * sizeof(char); + } + } + else { + size = item_count * get_size_by_dtype(pdc_type); + } + } + return size; +} + +static const char * +get_name_by_dtype(pdc_c_var_type_t type) +{ + if (type < 0 || type >= PDC_TYPE_COUNT) { + return NULL; + } + return DataTypeNames[type]; +} + +static pdc_c_var_type_t +get_dtype_by_enum_name(const char *enumName) +{ + for (int i = 0; i < PDC_TYPE_COUNT; i++) { + if (strcmp(DataTypeEnumNames[i], enumName) == 0) { + return (pdc_c_var_type_t)i; + } + } + return PDC_UNKNOWN; // assuming PDC_UNKNOWN is the enum value for "unknown" +} + +#endif /* PDC_GENERIC_H */ \ No newline at end of file diff --git a/src/commons/serde/include/pdc_serde.h b/src/commons/serde/include/pdc_serde.h new file mode 100644 index 000000000..6211e1917 --- /dev/null +++ b/src/commons/serde/include/pdc_serde.h @@ -0,0 +1,155 @@ +#ifndef PDC_SERDE_H +#define PDC_SERDE_H + +#include +#include +#include +#include "pdc_generic.h" + +#define MAX_KEYS 10 +#define MAX_BUFFER_SIZE 1000 + +typedef struct { + PDC_CType pdc_type; /**< Data type of the key */ + size_t size; /**< Size of the key */ + void * key; /**< Pointer to the key data */ +} PDC_SERDE_Key; + +typedef struct { + PDC_CType_Class pdc_class; /**< Class of the value */ + PDC_CType pdc_type; /**< Data type of the value */ + size_t size; // size of the data. If a string, it is strlen(data) + 1; + // if an array, it is the number of elements; + // if a struct, it is the totalSize of the data chunk of the struct, etc. + void *data; /**< Pointer to the value data */ +} PDC_SERDE_Value; + +typedef struct { + PDC_SERDE_Key *keys; /**< Array of keys */ + size_t numKeys; /**< Number of keys */ + size_t totalSize; /**< Total size of the header */ +} PDC_SERDE_Header; + +typedef struct { + size_t numValues; /**< Number of values */ + PDC_SERDE_Value *values; /**< Array of values */ + size_t totalSize; /**< Total size of the data */ +} PDC_SERDE_Data; + +typedef struct { + PDC_SERDE_Header *header; /**< Pointer to the header */ + PDC_SERDE_Data * data; /**< Pointer to the data */ + size_t totalSize; /**< Total size of the serialized data */ +} PDC_SERDE_SerializedData; + +/** + * @brief Initialize a serialized data structure + * + * @param initial_field_count Number of initial fields to allocate space for + * + * @return Pointer to the initialized PDC_SERDE_SerializedData structure + */ +PDC_SERDE_SerializedData *pdc_serde_init(int initial_field_count); + +/** + * @brief Append a key-value pair to the serialized data structure + * + * @param data Pointer to the PDC_SERDE_SerializedData structure + * @param key Pointer to the PDC_SERDE_Key structure representing the key + * @param value Pointer to the PDC_SERDE_Value structure representing the value + */ +void pdc_serde_append_key_value(PDC_SERDE_SerializedData *data, PDC_SERDE_Key *key, PDC_SERDE_Value *value); + +/** + * @brief get the total size of PDC_SERDE_SerializedData structure instance + * + * @param data Pointer to the PDC_SERDE_SerializedData structure instance + * + * @return total size of the PDC_SERDE_SerializedData structure instance +*/ +size_t get_total_size_for_serialized_data(PDC_SERDE_SerializedData *data); + +/** + * @brief Serialize the data in the serialized data structure and return the buffer + * + * @param data Pointer to the PDC_SERDE_SerializedData structure + * + * @return Pointer to the buffer containing the serialized data + */ +void *pdc_serde_serialize(PDC_SERDE_SerializedData *data); + +/** + * @brief Deserialize the buffer and return the deserialized data structure + * + * @param buffer Pointer to the buffer containing the serialized data + * + * @return Pointer to the deserialized PDC_SERDE_SerializedData structure + */ +PDC_SERDE_SerializedData *pdc_serde_deserialize(void *buffer); + +/** + * @brief Free the memory allocated for the serialized data structure + * + * @param data Pointer to the PDC_SERDE_SerializedData structure to be freed + */ +void pdc_serde_free(PDC_SERDE_SerializedData *data); + +/** + * @brief Print the contents of the serialized data structure + * + * @param data Pointer to the PDC_SERDE_SerializedData structure to be printed + */ +void pdc_serde_print(PDC_SERDE_SerializedData *data); + +/** + * @brief Create a PDC_SERDE_Key structure + * + * @param key Pointer to the key data + * @param pdc_type Data type of the key. For SERDE_Key, we only support PDC_CLS_SCALAR class. + * @param size Size of the key data + * + * @return Pointer to the created PDC_SERDE_Key structure + */ +static inline PDC_SERDE_Key * +PDC_SERDE_KEY(void *key, PDC_CType pdc_type, size_t size) +{ + PDC_SERDE_Key *pdc_key = (PDC_SERDE_Key *)malloc(sizeof(PDC_SERDE_Key)); + size_t key_size = (size_t) get_size_by_class_n_type(key, size, PDC_CLS_SCALAR, pdc_type); + pdc_key->key = malloc(key_size); + memcpy(pdc_key->key, key, key_size); + pdc_key->pdc_type = pdc_type; + pdc_key->size = key_size; + return pdc_key; +} + +/** + * @brief Create a PDC_SERDE_Value structure + * + * @param data Pointer to the value data + * @param pdc_type Data type of the value + * @param pdc_class Class of the value + * @param size Size of the value data + * + * @return Pointer to the created PDC_SERDE_Value structure + */ +static inline PDC_SERDE_Value * +PDC_SERDE_VALUE(void *data, PDC_CType pdc_type, PDC_CType_Class pdc_class, size_t size) +{ + PDC_SERDE_Value *pdc_value = (PDC_SERDE_Value *)malloc(sizeof(PDC_SERDE_Value)); + size_t value_size = 0; + if (pdc_class == PDC_CLS_STRUCT) { + // TODO: we need to check if data is a valid PDC_SERDE_SerializedData structure. + PDC_SERDE_SerializedData *struct_data = (PDC_SERDE_SerializedData *)data; + size = struct_data->totalSize; + } else { + value_size = (size_t) get_size_by_class_n_type(data, size, pdc_class, pdc_type); + } + pdc_value->data = malloc(value_size); + memcpy(pdc_value->data, data, value_size); + pdc_value->pdc_class = pdc_class; + pdc_value->pdc_type = pdc_type; + pdc_value->size = value_size; + return pdc_value; +} + +#endif /* PDC_SERDE_H */ \ No newline at end of file diff --git a/src/commons/serde/pdc_serde.c b/src/commons/serde/pdc_serde.c new file mode 100644 index 000000000..e0959b5cd --- /dev/null +++ b/src/commons/serde/pdc_serde.c @@ -0,0 +1,346 @@ +#include "pdc_serde.h" + +PDC_SERDE_SerializedData * +pdc_serde_init(int initial_field_count) +{ + PDC_SERDE_SerializedData *data = malloc(sizeof(PDC_SERDE_SerializedData)); + data->header = malloc(sizeof(PDC_SERDE_Header)); + data->header->keys = malloc(sizeof(PDC_SERDE_Key) * initial_field_count); + data->header->numKeys = 0; + data->header->totalSize = 0; + data->data = malloc(sizeof(PDC_SERDE_Data)); + data->data->values = malloc(sizeof(PDC_SERDE_Value) * initial_field_count); + data->data->numValues = 0; + data->data->totalSize = 0; + return data; +} + +void +pdc_serde_append_key_value(PDC_SERDE_SerializedData *data, PDC_SERDE_Key *key, PDC_SERDE_Value *value) +{ + data->header->keys[data->header->numKeys] = *key; + data->header->numKeys++; + // append type, size, and key + data->header->totalSize += (sizeof(int) + sizeof(size_t) + key->size); + data->data->values[data->data->numValues] = *value; + data->data->numValues++; + // append class, type, size, and data + data->data->totalSize += (sizeof(int) + sizeof(int) + sizeof(size_t) + value->size); +} + +size_t get_total_size_for_serialized_data(PDC_SERDE_SerializedData *data) { + if (data->totalSize <= 0) { + size_t total_size = data->header->totalSize + data->data->totalSize + sizeof(size_t) * 6; + data->totalSize = total_size; + } + return data->totalSize; +} + +// clang-format off +/** + * This function serializes the entire PDC_SERDE_SerializedData structure. + * + * The overview of the serialized binary data layout is: + * +---------------------+---------------------+----------------------+---------------------+----------------------+----------------------+----------------------+----------------------+ + * | Size of the Header | Size of the Data | Number of Keys | Header Region | Data Offset | Number of Values | Data Region | Data Offset | + * | (size_t) | (size_t) | (size_t) | | (size_t) | (size_t) | | (size_t) | + * +---------------------+---------------------+----------------------+---------------------+----------------------+----------------------+----------------------+----------------------+ + * + * The first 2 field is called meta-header, which provides metadata about size of the header region and the size of the data region. + * Note that the size of the header region doesn't include the 'Number of Keys' field. + * Also, the size of the data region doesn't include the 'Data Offset' field. + * + * Then the following is the header region with two keys: + * +-----------------------+-------------------------+-----------------------------+---------------------------+--------------------------+-----------------------------+---------------------------+ + * | Number of Keys | Key 1 Type | Key 1 Size | Key 1 Data | Key 2 Type | Key 2 Size | Key 2 Data | + * | (size_t) | (int8_t) | (size_t) | (Variable size depending | (int8_t) | (size_t) | (Variable size depending | + * | | | | on Key 1 Size) | | | on Key 2 Size) | + * +-----------------------+-------------------------+-----------------------------+---------------------------+--------------------------+-----------------------------+---------------------------+ + * + * Then, the following is a header offset validation point and the data region with the final offset validation point. + * + * |----------------------------------------------------------------------------------------------------------| + * | Data Offset (size_t) | Number of Value Entries (size_t) | Value 1 Class (int8_t) | Value 1 Type (int8_t) | + * |----------------------------------------------------------------------------------------------------------| + * | Value 1 Size (size_t)| Value 1 Data (Variable size depending on Value 1 Size) | Value 2 Class (int8_t) | + * |----------------------------------------------------------------------------------------------------------| + * | Value 2 Type (int8_t)| Value 2 Size (size_t) | Value 2 Data (Variable size depending on Value 2 Size) | + * |----------------------------------------------------------------------------------------------------------| + * | ...repeated for the number of value entries in the data... | + * |----------------------------------------------------------------------------------------------------------| + * | Final Data Offset (size_t) | + * |----------------------------------------------------------------------------------------------------------| + * + * Please refer to `get_size_by_class_n_type` function in pdc_generic.h for size calculation on scalar values and array values. + * + */ +// clang-format on +void * +pdc_serde_serialize(PDC_SERDE_SerializedData *data) +{ + // The buffer contains: + // the size of the header (size_t) + + // the size of the data (size_t) + + // the number of keys (size_t) + + // the header region + + // the data offset (size_t) + + // the number of value entries (size_t) + + // the data region + void *buffer = malloc(get_total_size_for_serialized_data(data)); + // serialize the meta header, which contains only the size of the header and the size of the data region. + memcpy(buffer, &data->header->totalSize, sizeof(size_t)); + memcpy(buffer + sizeof(size_t), &data->data->totalSize, sizeof(size_t)); + + // serialize the header + // start with the number of keys + memcpy(buffer + sizeof(size_t) * 2, &data->header->numKeys, sizeof(size_t)); + // then the keys + size_t offset = sizeof(size_t) * 3; + for (int i = 0; i < data->header->numKeys; i++) { + int8_t pdc_type = (int8_t)(data->header->keys[i].pdc_type); + memcpy(buffer + offset, &pdc_type, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(buffer + offset, &data->header->keys[i].size, sizeof(size_t)); + offset += sizeof(size_t); + memcpy(buffer + offset, data->header->keys[i].key, data->header->keys[i].size); + offset += data->header->keys[i].size; + } + + // serialize the data offset, this is for validation purpose to see if header region is corrupted. + memcpy(buffer + offset, &offset, sizeof(size_t)); + offset += sizeof(size_t); + + // serialize the data + // start with the number of value entries + memcpy(buffer + offset, &data->data->numValues, sizeof(size_t)); + offset += sizeof(size_t); + // then the values + for (int i = 0; i < data->data->numValues; i++) { + int8_t pdc_class = (int8_t)data->data->values[i].pdc_class; + int8_t pdc_type = (int8_t)data->data->values[i].pdc_type; + memcpy(buffer + offset, &pdc_class, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(buffer + offset, &pdc_type, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(buffer + offset, &data->data->values[i].size, sizeof(size_t)); + offset += sizeof(size_t); + memcpy(buffer + offset, data->data->values[i].data, data->data->values[i].size); + offset += data->data->values[i].size; + } + // serialize the data offset again, this is for validation purpose to see if data region is corrupted. + memcpy(buffer + offset, &offset, sizeof(size_t)); + offset += sizeof(size_t); + return buffer; +} + +PDC_SERDE_SerializedData * +pdc_serde_deserialize(void *buffer) +{ + size_t offset = 0; + // read the meta header + size_t headerSize; + size_t dataSize; + memcpy(&headerSize, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + memcpy(&dataSize, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + + // read the header + size_t numKeys; + memcpy(&numKeys, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + PDC_SERDE_Header *header = malloc(sizeof(PDC_SERDE_Header)); + header->keys = malloc(sizeof(PDC_SERDE_Key) * numKeys); + header->numKeys = numKeys; + header->totalSize = headerSize; + for (int i = 0; i < numKeys; i++) { + int8_t pdc_type; + size_t size; + memcpy(&pdc_type, buffer + offset, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(&size, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + void *key = malloc(size); + memcpy(key, buffer + offset, size); + offset += size; + header->keys[i].key = key; + header->keys[i].pdc_type = (PDC_CType)pdc_type; + header->keys[i].size = size; + } + + // read the data offset + size_t dataOffset; + memcpy(&dataOffset, buffer + offset, sizeof(size_t)); + // check the data offset + if (dataOffset != offset) { + printf("Error: data offset does not match the expected offset.\n"); + return NULL; + } + offset += sizeof(size_t); + + // read the data + size_t numValues; + memcpy(&numValues, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + PDC_SERDE_Data *data = malloc(sizeof(PDC_SERDE_Data)); + data->values = malloc(sizeof(PDC_SERDE_Value) * numValues); + data->numValues = numValues; + data->totalSize = dataSize; + for (int i = 0; i < numValues; i++) { + int8_t pdc_class; + int8_t pdc_type; + size_t size; + memcpy(&pdc_class, buffer + offset, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(&pdc_type, buffer + offset, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(&size, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + void *value = malloc(size); + memcpy(value, buffer + offset, size); + offset += size; + data->values[i].data = value; + data->values[i].pdc_class = (PDC_CType_Class)pdc_class; + data->values[i].pdc_type = (PDC_CType)pdc_type; + data->values[i].size = size; + } + // check the total size + memcpy(&dataOffset, buffer + offset, sizeof(size_t)); + // check the data offset + if (dataOffset != offset) { + printf("Error: data offset does not match the expected offset.\n"); + return NULL; + } + offset += sizeof(size_t); + if (offset != headerSize + sizeof(size_t) * 6 + dataSize) { + printf("Error: total size does not match the expected size.\n"); + return NULL; + } + // create the serialized data + PDC_SERDE_SerializedData *serializedData = malloc(sizeof(PDC_SERDE_SerializedData)); + serializedData->header = header; + serializedData->data = data; + serializedData->totalSize = headerSize + dataSize + sizeof(size_t) * 6; + + return serializedData; +} + +void +pdc_serde_free(PDC_SERDE_SerializedData *data) +{ + for (int i = 0; i < data->header->numKeys; i++) { + free(data->header->keys[i].key); + } + free(data->header->keys); + for (int i = 0; i < data->data->numValues; i++) { + free(data->data->values[i].data); + } + free(data->data->values); + free(data->header); + free(data->data); + free(data); +} + +void +pdc_serde_print(PDC_SERDE_SerializedData *data) +{ + printf("Header:\n"); + printf(" numKeys: %zu\n", data->header->numKeys); + printf(" totalSize: %zu\n", data->header->totalSize); + for (int i = 0; i < data->header->numKeys; i++) { + printf(" key %d:\n", i); + printf(" type: %d\n", data->header->keys[i].pdc_type); + printf(" size: %zu\n", data->header->keys[i].size); + printf(" key: %s\n", (char *)data->header->keys[i].key); + } + printf("Data:\n"); + printf(" numValues: %zu\n", data->data->numValues); + printf(" totalSize: %zu\n", data->data->totalSize); + for (int i = 0; i < data->data->numValues; i++) { + printf(" value %d:\n", i); + printf(" class: %d\n", data->data->values[i].pdc_class); + printf(" type: %d\n", data->data->values[i].pdc_type); + printf(" size: %zu\n", data->data->values[i].size); + printf(" data: "); + if (data->data->values[i].pdc_class == PDC_STRING) { + printf("%s\n", (char *)data->data->values[i].data); + } + else { + printf("\n"); + } + } +} + +int +test_serde_framework() +{ + // Initialize a serialized data structure + PDC_SERDE_SerializedData *data = pdc_serde_init(5); + + // Create and append key-value pairs for different data types + char * intKey_str = "int"; + int intVal = 42; + PDC_SERDE_Key * intKey = PDC_SERDE_KEY(intKey_str, PDC_STRING, sizeof(intKey_str)); + PDC_SERDE_Value *intValue = PDC_SERDE_VALUE(&intVal, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); + pdc_serde_append_key_value(data, intKey, intValue); + + char * doubleKey_str = "double"; + double doubleVal = 3.14159; + PDC_SERDE_Key * doubleKey = PDC_SERDE_KEY(doubleKey_str, PDC_STRING, sizeof(doubleKey_str)); + PDC_SERDE_Value *doubleValue = + PDC_SERDE_VALUE(&doubleVal, PDC_DOUBLE, PDC_CLS_SCALAR, sizeof(double)); + pdc_serde_append_key_value(data, doubleKey, doubleValue); + + char * strKey_str = "string"; + char * strVal = "Hello, World!"; + PDC_SERDE_Key *strKey = PDC_SERDE_KEY(strKey_str, PDC_STRING, (strlen(strKey_str) + 1) * sizeof(char)); + PDC_SERDE_Value *strValue = + PDC_SERDE_VALUE(strVal, PDC_STRING, PDC_CLS_SCALAR, (strlen(strVal) + 1) * sizeof(char)); + pdc_serde_append_key_value(data, strKey, strValue); + + char * arrayKey_str = "array"; + int intArray[3] = {1, 2, 3}; + PDC_SERDE_Key * arrayKey = PDC_SERDE_KEY(arrayKey_str, PDC_STRING, sizeof(arrayKey_str)); + PDC_SERDE_Value *arrayValue = PDC_SERDE_VALUE(intArray, PDC_INT, PDC_CLS_ARRAY, sizeof(int) * 3); + pdc_serde_append_key_value(data, arrayKey, arrayValue); + + typedef struct { + int x; + int y; + } Point; + + char *pointKey = "point"; + Point pointVal = {10, 20}; + + PDC_SERDE_SerializedData *point_data = pdc_serde_init(2); + PDC_SERDE_Key * x_name = PDC_SERDE_KEY("x", PDC_STRING, sizeof(char *)); + PDC_SERDE_Value *x_value = PDC_SERDE_VALUE(&pointVal.x, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); + + PDC_SERDE_Key * y_name = PDC_SERDE_KEY("y", PDC_STRING, sizeof(char *)); + PDC_SERDE_Value *y_value = PDC_SERDE_VALUE(&pointVal.y, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); + + pdc_serde_append_key_value(point_data, x_name, x_value); + pdc_serde_append_key_value(point_data, y_name, y_value); + void *point_buffer = pdc_serde_serialize(point_data); + + PDC_SERDE_Key * structKey = PDC_SERDE_KEY(pointKey, PDC_STRING, sizeof(pointKey)); + PDC_SERDE_Value *structValue = + PDC_SERDE_VALUE(point_buffer, PDC_VOID_PTR, PDC_CLS_STRUCT, sizeof(Point)); + pdc_serde_append_key_value(data, structKey, structValue); + + // Serialize the data + void *buffer = pdc_serde_serialize(data); + + // Deserialize the buffer + PDC_SERDE_SerializedData *deserializedData = pdc_serde_deserialize(buffer); + + // Print the deserialized data + pdc_serde_print(deserializedData); + + // Free the memory + pdc_serde_free(data); + pdc_serde_free(deserializedData); + free(buffer); + + return 0; +} \ No newline at end of file diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 449b64f10..abe3917c6 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -48,7 +48,6 @@ add_executable(pdc_server.exe ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c ${PDC_SOURCE_DIR}/src/utils/pdc_region_utils.c - ${PDC_SOURCE_DIR}/src/utils/pdc_timing.c ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_analysis_common.c ${PDC_SOURCE_DIR}/src/api/pdc_transform/pdc_transforms_common.c ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_hist_pkg.c @@ -70,9 +69,9 @@ endif() if(PDC_ENABLE_FASTBIT) message(STATUS "Enabled fastbit") - target_link_libraries(pdc_server.exe mercury pdcprof -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} ${FASTBIT_LIBRARY}/libfastbit.so) + target_link_libraries(pdc_server.exe mercury ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} ${FASTBIT_LIBRARY}/libfastbit.so) else() - target_link_libraries(pdc_server.exe mercury pdcprof -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES}) + target_link_libraries(pdc_server.exe mercury ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES}) endif() diff --git a/src/server/include/pdc_client_server_common.h b/src/server/include/pdc_client_server_common.h index a57e2169f..e125af9a5 100644 --- a/src/server/include/pdc_client_server_common.h +++ b/src/server/include/pdc_client_server_common.h @@ -1190,6 +1190,11 @@ hg_proc_pdc_kvtag_t(hg_proc_t proc, void *data) // HG_LOG_ERROR("Proc error"); return ret; } + ret = hg_proc_int8_t(proc, &struct_data->type); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } if (struct_data->size) { switch (hg_proc_get_op(proc)) { case HG_DECODE: diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index a0a7845a4..dbbff7bfd 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -3096,7 +3096,7 @@ HG_TEST_RPC_CB(region_release, handle) size2 = HG_Bulk_get_size(remote_bulk_handle); if (size != size2) { error = 1; - printf("==PDC_SERVER: local size %lu, remote %lu\n", size, size2); + printf("==PDC_SERVER: local size %llu, remote %llu\n", size, size2); PGOTO_ERROR(HG_OTHER_ERROR, "===PDC SERVER: HG_TEST_RPC_CB(region_release, " "handle) local and remote bulk size does not match"); } @@ -3269,7 +3269,7 @@ HG_TEST_RPC_CB(region_release, handle) size2 = HG_Bulk_get_size(remote_bulk_handle); if (size != size2) { error = 1; - printf("==PDC_SERVER: local size %lu, remote %lu\n", size, size2); + printf("==PDC_SERVER: local size %llu, remote %llu\n", size, size2); /* PGOTO_ERROR(HG_OTHER_ERROR, "===PDC SERVER: HG_TEST_RPC_CB(region_release, * handle) local and remote bulk size does not match"); */ } @@ -6849,7 +6849,7 @@ PDC_kvtag_dup(pdc_kvtag_t *from, pdc_kvtag_t **to) (*to) = (pdc_kvtag_t *)calloc(1, sizeof(pdc_kvtag_t)); (*to)->name = (char *)malloc(strlen(from->name) + 1); (*to)->size = from->size; - + (*to)->type = from->type; (*to)->value = (void *)malloc(from->size); memcpy((void *)(*to)->name, (void *)from->name, strlen(from->name) + 1); memcpy((void *)(*to)->value, (void *)from->value, from->size); diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index a3020628a..449b31c06 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1224,6 +1224,7 @@ PDC_Server_checkpoint() fwrite(&key_len, sizeof(int), 1, file); fwrite(kvlist_elt->kvtag->name, key_len, 1, file); fwrite(&kvlist_elt->kvtag->size, sizeof(uint32_t), 1, file); + fwrite(&kvlist_elt->kvtag->type, sizeof(int8_t), 1, file); fwrite(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, 1, file); } @@ -1403,7 +1404,8 @@ PDC_Server_restart(char *filename) } // init hash table - PDC_Server_init_hash_table(); + // FIXME: check if we need to init the hash table again. + // PDC_Server_init_hash_table(); if (fread(&n_cont, sizeof(int), 1, file) != 1) { printf("Read failed for n_count\n"); @@ -1492,6 +1494,9 @@ PDC_Server_restart(char *filename) if (fread(&kvtag_list->kvtag->size, sizeof(uint32_t), 1, file) != 1) { printf("Read failed for kvtag_list->kvtag->size\n"); } + if (fread(&kvtag_list->kvtag->type, sizeof(int8_t), 1, file) != 1) { + printf("Read failed for kvtag_list->kvtag->type\n"); + } kvtag_list->kvtag->value = malloc(kvtag_list->kvtag->size); if (fread(kvtag_list->kvtag->value, kvtag_list->kvtag->size, 1, file) != 1) { printf("Read failed for kvtag_list->kvtag->value\n"); diff --git a/src/server/pdc_server_region/pdc_server_region_request_handler.h b/src/server/pdc_server_region/pdc_server_region_request_handler.h index f3dc64cde..04ab48481 100644 --- a/src/server/pdc_server_region/pdc_server_region_request_handler.h +++ b/src/server/pdc_server_region/pdc_server_region_request_handler.h @@ -811,7 +811,7 @@ HG_TEST_RPC_CB(transfer_request, handle) ret_value = HG_Respond(handle, NULL, NULL, &out); if (in.access_type == PDC_WRITE) { ret_value = HG_Bulk_create(info->hg_class, 1, &(local_bulk_args->data_buf), - &(local_bulk_args->total_mem_size), HG_BULK_READWRITE, + (const hg_size_t *)&(local_bulk_args->total_mem_size), HG_BULK_READWRITE, &(local_bulk_args->bulk_handle)); if (ret_value != HG_SUCCESS) { printf("Error at HG_TEST_RPC_CB(transfer_request, handle): @ line %d \n", __LINE__); @@ -881,7 +881,7 @@ HG_TEST_RPC_CB(transfer_request, handle) *((int *)(local_bulk_args->data_buf + sizeof(int)))); */ ret_value = HG_Bulk_create(info->hg_class, 1, &(local_bulk_args->data_buf), - &(local_bulk_args->total_mem_size), HG_BULK_READWRITE, + (const hg_size_t *)&(local_bulk_args->total_mem_size), HG_BULK_READWRITE, &(local_bulk_args->bulk_handle)); if (ret_value != HG_SUCCESS) { printf("Error at HG_TEST_RPC_CB(transfer_request, handle): @ line %d \n", __LINE__); diff --git a/src/tests/cont_tags.c b/src/tests/cont_tags.c index 04132c332..f6c961a71 100644 --- a/src/tests/cont_tags.c +++ b/src/tests/cont_tags.c @@ -37,6 +37,7 @@ main(int argc, char **argv) int rank = 0, size = 1; char tag_value[128], tag_value2[128], *tag_value_ret; + pdc_var_type_t value_type; psize_t value_size; strcpy(tag_value, "some tag value"); strcpy(tag_value2, "some tag value 2 is longer than tag 1"); @@ -78,31 +79,31 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCcont_put_tag(cont, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCcont_put_tag(cont, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 1\n"); ret_value = 1; } - ret = PDCcont_put_tag(cont, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCcont_put_tag(cont, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 1\n"); ret_value = 1; } - ret = PDCcont_put_tag(cont2, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCcont_put_tag(cont2, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 2\n"); ret_value = 1; } - ret = PDCcont_put_tag(cont2, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCcont_put_tag(cont2, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 2\n"); ret_value = 1; } - ret = PDCcont_get_tag(cont, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 1\n"); ret_value = 1; @@ -112,7 +113,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCcont_get_tag(cont, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 1\n"); ret_value = 1; @@ -123,7 +124,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCcont_get_tag(cont2, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont2, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 2\n"); ret_value = 1; @@ -134,7 +135,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCcont_get_tag(cont2, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont2, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 2\n"); ret_value = 1; diff --git a/src/tests/kvtag_add_get.c b/src/tests/kvtag_add_get.c index 97eadffed..91686b9be 100644 --- a/src/tests/kvtag_add_get.c +++ b/src/tests/kvtag_add_get.c @@ -33,13 +33,14 @@ int main() { - pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; - pdc_kvtag_t kvtag1, kvtag2, kvtag3; - char * v1 = "value1"; - int v2 = 2; - double v3 = 3.45; - void * value1, *value2, *value3; - psize_t value_size; + pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; + pdc_kvtag_t kvtag1, kvtag2, kvtag3; + char * v1 = "value1"; + int v2 = 2; + double v3 = 3.45; + pdc_var_type_t type1, type2, type3; + void * value1, *value2, *value3; + psize_t value_size; // create a pdc pdc = PDCinit("pdc"); @@ -88,42 +89,45 @@ main() kvtag1.name = "key1string"; kvtag1.value = (void *)v1; + kvtag1.type = PDC_STRING; kvtag1.size = strlen(v1) + 1; kvtag2.name = "key2int"; kvtag2.value = (void *)&v2; + kvtag1.type = PDC_INT; kvtag2.size = sizeof(int); kvtag3.name = "key3double"; kvtag3.value = (void *)&v3; + kvtag1.type = PDC_DOUBLE; kvtag3.size = sizeof(double); - if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.size) < 0) + if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.type, kvtag1.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.size) < 0) + if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.type, kvtag2.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.size) < 0) + if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.type, kvtag3.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", kvtag1.name, (char *)value1); - if (PDCobj_get_tag(obj2, kvtag2.name, (void *)&value2, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, kvtag2.name, (void *)&value2, (void *)&type2, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%d] from o2\n", kvtag2.name, *(int *)value2); - if (PDCobj_get_tag(obj2, kvtag3.name, (void *)&value3, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, kvtag3.name, (void *)&value3, (void *)&type3, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%f] from o2\n", kvtag3.name, *(double *)value3); @@ -135,15 +139,16 @@ main() v1 = "New Value After Delete"; kvtag1.value = (void *)v1; + kvtag1.type = PDC_STRING; kvtag1.size = strlen(v1) + 1; - if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.size) < 0) + if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.type, kvtag1.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); /* PDC_free_kvtag(&value1); */ - if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", kvtag1.name, (char *)value1); diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index ee5efdcbf..2ccc67828 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -220,7 +220,7 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value v = i + my_obj_s; for (j = 0; j < n_attr; j++) { sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", v, j); - if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], tag_value_len + 1) < 0) + if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], PDC_STRING, tag_value_len + 1) < 0) printf("fail to add a kvtag to o%" PRIu64 "\n", v); } } @@ -240,10 +240,11 @@ get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, uint64_t n_attr, void **tag { uint64_t i; char tag_name[256]; + pdc_var_type_t tag_type; for (i = 0; i < n_attr; i++) { sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", obj_name_v, i); - if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], (void *)&value_size[i]) < 0) + if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], &tag_type, (void *)&value_size[i]) < 0) printf("fail to get a kvtag from o%" PRIu64 "\n", obj_name_v); } } diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index 280eda25b..d145560d0 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -76,7 +76,8 @@ main(int argc, char *argv[]) double stime, total_time, percent_time; pdc_kvtag_t kvtag; void ** values; - size_t value_size; + pdc_var_type_t value_type; + size_t value_size; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); @@ -167,6 +168,7 @@ main(int argc, char *argv[]) // Add tags kvtag.name = "Group"; kvtag.value = (void *)&v; + kvtag.type = PDC_INT; kvtag.size = sizeof(int); #ifdef ENABLE_MPI @@ -175,7 +177,7 @@ main(int argc, char *argv[]) #endif for (i = 0; i < my_add_tag; i++) { v = i + my_add_tag_s; - if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) printf("fail to add a kvtag to o%d\n", i + my_obj_s); if (i % tag_1percent == 0) { @@ -208,7 +210,7 @@ main(int argc, char *argv[]) stime = MPI_Wtime(); #endif for (i = 0; i < my_query; i++) { - if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_size) < 0) + if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, (void *)&value_size) < 0) printf("fail to get a kvtag from o%d\n", i + my_query_s); if (i % query_1percent == 0) { diff --git a/src/tests/kvtag_get.c b/src/tests/kvtag_get.c index a8fcf70f1..7ca1e7c0e 100644 --- a/src/tests/kvtag_get.c +++ b/src/tests/kvtag_get.c @@ -33,9 +33,10 @@ int main() { - pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; - pdc_kvtag_t *value1, *value2, *value3; - psize_t value_size; + pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; + pdc_kvtag_t * value1, *value2, *value3; + pdc_var_type_t type1, type2, type3; + psize_t value_size; // create a pdc pdc = PDCinit("pdc"); @@ -82,17 +83,17 @@ main() else printf("Fail to create object @ line %d!\n", __LINE__); - if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", value1->name, (char *)value1->value); - if (PDCobj_get_tag(obj2, "key2int", (void *)&value2, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, "key2int", (void *)&value2, (void *)&type2, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%d] from o2\n", value2->name, *(int *)value2->value); - if (PDCobj_get_tag(obj2, "key3double", (void *)&value3, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, "key3double", (void *)&value3, (void *)&type3, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%f] from o2\n", value3->name, @@ -100,7 +101,7 @@ main() PDC_free_kvtag(&value1); - if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", value1->name, (char *)value1->value); diff --git a/src/tests/kvtag_query.c b/src/tests/kvtag_query.c index 88e6621d7..cf1e80dcb 100644 --- a/src/tests/kvtag_query.c +++ b/src/tests/kvtag_query.c @@ -91,32 +91,35 @@ main() kvtag1.name = "key1string"; kvtag1.value = (void *)v1; + kvtag1.type = PDC_STRING; kvtag1.size = strlen(v1) + 1; kvtag2.name = "key2int"; kvtag2.value = (void *)&v2; + kvtag2.type = PDC_INT; kvtag2.size = sizeof(int); kvtag3.name = "key3double"; kvtag3.value = (void *)&v3; + kvtag3.type = PDC_DOUBLE; kvtag3.size = sizeof(double); - if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.size) < 0) + if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.type, kvtag1.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj1, kvtag2.name, kvtag2.value, kvtag2.size) < 0) + if (PDCobj_put_tag(obj1, kvtag2.name, kvtag2.value, kvtag2.type, kvtag2.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.size) < 0) + if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.type, kvtag2.size) < 0) printf("fail to add a kvtag to o2\n"); else printf("successfully added a kvtag to o2\n"); - if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.size) < 0) + if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.type, kvtag3.size) < 0) printf("fail to add a kvtag to o2\n"); else printf("successfully added a kvtag to o2\n"); diff --git a/src/tests/kvtag_query_scale.c b/src/tests/kvtag_query_scale.c index a42cd5a5a..b9e4a6ffa 100644 --- a/src/tests/kvtag_query_scale.c +++ b/src/tests/kvtag_query_scale.c @@ -129,6 +129,7 @@ main(int argc, char *argv[]) // Add tags kvtag.name = "Group"; kvtag.value = (void *)&v; + kvtag.type = PDC_INT; kvtag.size = sizeof(int); for (iter = 0; iter < round; iter++) { @@ -136,7 +137,7 @@ main(int argc, char *argv[]) v = iter; for (i = 0; i < my_add_tag; i++) { - if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) printf("fail to add a kvtag to o%d\n", i + my_obj_s); } @@ -153,6 +154,7 @@ main(int argc, char *argv[]) kvtag.name = "Group"; kvtag.value = (void *)&v; + kvtag.type = PDC_INT; kvtag.size = sizeof(int); for (iter = 0; iter < round; iter++) { diff --git a/src/tests/obj_tags.c b/src/tests/obj_tags.c index 2651cf64e..db99bec1b 100644 --- a/src/tests/obj_tags.c +++ b/src/tests/obj_tags.c @@ -44,6 +44,7 @@ main(int argc, char **argv) dims[2] = 4; char tag_value[128], tag_value2[128], *tag_value_ret; char cont_name[128], obj_name1[128], obj_name2[128]; + pdc_var_type_t value_type; psize_t value_size; strcpy(tag_value, "some tag value"); @@ -119,30 +120,30 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_put_tag(obj1, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCobj_put_tag(obj1, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 1\n"); ret_value = 1; } - ret = PDCobj_put_tag(obj1, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCobj_put_tag(obj1, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 1\n"); ret_value = 1; } - ret = PDCobj_put_tag(obj2, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCobj_put_tag(obj2, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 2\n"); ret_value = 1; } - ret = PDCobj_put_tag(obj2, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCobj_put_tag(obj2, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 2\n"); ret_value = 1; } - ret = PDCobj_get_tag(obj1, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj1, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 1\n"); ret_value = 1; @@ -153,7 +154,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_get_tag(obj1, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj1, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 1\n"); ret_value = 1; @@ -164,7 +165,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_get_tag(obj2, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj2, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 2\n"); ret_value = 1; @@ -175,7 +176,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_get_tag(obj2, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj2, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 2\n"); ret_value = 1; diff --git a/src/tests/pdc_transforms_lib.c b/src/tests/pdc_transforms_lib.c index f1a83f742..6397bdc70 100644 --- a/src/tests/pdc_transforms_lib.c +++ b/src/tests/pdc_transforms_lib.c @@ -6,21 +6,26 @@ * >> pdc_public.h * * typedef enum { - * PDC_UNKNOWN = -1, - * PDC_INT = 0, - * PDC_FLOAT = 1, - * PDC_DOUBLE = 2, - * PDC_STRING = 3, - * PDC_COMPOUND = 4, - * PDC_ENUM = 5, - * PDC_ARRAY = 6, - * PDC_UINT = 7, - * PDC_INT64 = 8, - * PDC_UINT64 = 9, - * PDC_INT16 = 10, - * PDC_INT8 = 11, - * NCLASSES = 12 - * } PDC_var_type_t; + * PDC_UNKNOWN = -1, * error * + * PDC_INT = 0, * integer types (identical to int32_t) * + * PDC_FLOAT = 1, * floating-point types * + * PDC_DOUBLE = 2, * double types * + * PDC_CHAR = 3, * character types * + * PDC_STRING = 4, * string types * + * PDC_BOOLEAN = 5, * boolean types * + * PDC_SHORT = 6, * short types * + * PDC_UINT = 7, * unsigned integer types (identical to uint32_t) * + * PDC_INT64 = 8, * 64-bit integer types * + * PDC_UINT64 = 9, * 64-bit unsigned integer types * + * PDC_INT16 = 10, * 16-bit integer types * + * PDC_INT8 = 11, * 8-bit integer types * + * PDC_UINT8 = 12, * 8-bit unsigned integer types * + * PDC_UINT16 = 13, * 16-bit unsigned integer types * + * PDC_LONG = 14, * long types * + * PDC_VOID_PTR = 15, * void pointer type * + * PDC_SIZE_T = 16, * size_t type * + * TYPE_COUNT = 17 * this is the number of var types and has to be the last * + * } pdc_c_var_type_t; */ static int diff --git a/src/utils/include/pdc_id_pkg.h b/src/utils/include/pdc_id_pkg.h deleted file mode 100644 index 9623661fd..000000000 --- a/src/utils/include/pdc_id_pkg.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#ifndef PDC_ID_PKG_H -#define PDC_ID_PKG_H - -#include "pdc_private.h" -#include "pdc_linkedlist.h" -#include "mercury_atomic.h" -/* - * Number of bits to use for ID Type in each atom. Increase if more types - * are needed (though this will decrease the number of available IDs per - * type). This is the only number that must be changed since all other bit - * field sizes and masks are calculated from TYPE_BITS. - */ -#define TYPE_BITS 8 -#define TYPE_MASK (((pdcid_t)1 << TYPE_BITS) - 1) -#define PDC_MAX_NUM_TYPES TYPE_MASK -/* - * Number of bits to use for the Atom index in each atom (assumes 8-bit - * bytes). We don't use the sign bit. - */ -#define ID_BITS ((sizeof(pdcid_t) * 8) - (TYPE_BITS + 1)) -#define ID_MASK (((pdcid_t)1 << ID_BITS) - 1) - -/* Map an atom to an ID type number */ -#define PDC_TYPE(a) ((PDC_type_t)(((pdcid_t)(a) >> ID_BITS) & TYPE_MASK)) - -struct _pdc_id_info { - pdcid_t id; /* ID for this info */ - hg_atomic_int32_t count; /* ref. count for this atom */ - void * obj_ptr; /* pointer associated with the atom */ - PDC_LIST_ENTRY(_pdc_id_info) entry; -}; - -#endif /* PDC_ID_PKG_H */ diff --git a/src/utils/include/pdc_linkedlist.h b/src/utils/include/pdc_linkedlist.h deleted file mode 100644 index 6fa7ce08a..000000000 --- a/src/utils/include/pdc_linkedlist.h +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (C) 2013-2016 Argonne National Laboratory, Department of Energy, - * UChicago Argonne, LLC and The HDF Group. - * All rights reserved. - * - * The full copyright notice, including terms governing use, modification, - * and redistribution, is contained in the COPYING file that can be - * found at the root of the source code distribution tree. - */ - -/* Code below is derived from sys/queue.h which follows the below notice: - * - * Copyright (c) 1991, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * @(#)queue.h 8.5 (Berkeley) 8/20/94 - */ - -#ifndef PDC_LINKEDLIST_H -#define PDC_LINKEDLIST_H - -#include "pdc_cont_pkg.h" -#include "pdc_cont.h" -#include "mercury_thread_mutex.h" -#include - -#define PDC_LIST_HEAD_INITIALIZER(name) \ - { \ - NULL \ - } - -#define PDC_LIST_HEAD_INIT(struct_head_name, var_name) \ - struct struct_head_name var_name = PDC_LIST_HEAD_INITIALIZER(var_name) - -#define PDC_LIST_HEAD_DECL(struct_head_name, struct_entry_name) \ - struct struct_head_name { \ - struct struct_entry_name *head; \ - } - -#define PDC_LIST_HEAD(struct_entry_name) \ - struct { \ - struct struct_entry_name *head; \ - hg_thread_mutex_t lock; \ - } - -#define PDC_LIST_ENTRY(struct_entry_name) \ - struct { \ - struct struct_entry_name * next; \ - struct struct_entry_name **prev; \ - } - -#define PDC_LIST_INIT(head_ptr) \ - do { \ - (head_ptr)->head = NULL; \ - hg_thread_mutex_init(&(head_ptr)->lock); \ - } while (/*CONSTCOND*/ 0) - -#define PDC_LIST_IS_EMPTY(head_ptr) ((head_ptr)->head == NULL) - -#define PDC_LIST_FIRST(head_ptr) ((head_ptr)->head) - -#define PDC_LIST_GET_FIRST(var, head_ptr) (var = (head_ptr)->head) - -#define PDC_LIST_NEXT(entry_ptr, entry_field_name) ((entry_ptr)->entry_field_name.next) - -#define PDC_LIST_TO_NEXT(entry_ptr, entry_field_name) ((entry_ptr) = (entry_ptr)->entry_field_name.next) - -#define PDC_LIST_INSERT_HEAD(head_ptr, entry_ptr, entry_field_name) \ - do { \ - if (((entry_ptr)->entry_field_name.next = (head_ptr)->head) != NULL) \ - (head_ptr)->head->entry_field_name.prev = &(entry_ptr)->entry_field_name.next; \ - (head_ptr)->head = (entry_ptr); \ - (entry_ptr)->entry_field_name.prev = &(head_ptr)->head; \ - } while (/*CONSTCOND*/ 0) - -/* TODO would be nice to not have any condition */ -#define PDC_LIST_REMOVE(entry_ptr, entry_field_name) \ - do { \ - if ((entry_ptr)->entry_field_name.next != NULL) \ - (entry_ptr)->entry_field_name.next->entry_field_name.prev = (entry_ptr)->entry_field_name.prev; \ - *(entry_ptr)->entry_field_name.prev = (entry_ptr)->entry_field_name.next; \ - } while (/*CONSTCOND*/ 0) - -#define PDC_LIST_FOREACH(var, head_ptr, entry_field_name) \ - for ((var) = ((head_ptr)->head); (var); (var) = ((var)->entry_field_name.next)) - -#define PDC_LIST_SEARCH(var, head_ptr, entry_field_name, item, value) \ - for ((var) = ((head_ptr)->head); (((var)->item != value) && (var)); \ - (var) = ((var)->entry_field_name.next)) - -#define PDC_LIST_SEARCH_CONT_NAME(var, head_ptr, entry_field_name, member, n, name) \ - for ((var) = ((head_ptr)->head); \ - ((var) && strcmp(((struct _pdc_cont_info *)((var)->member))->cont_info_pub->n, name) != 0); \ - (var) = ((var)->entry_field_name.next)) - -#endif /* PDC_LINKEDLIST_H */ diff --git a/src/utils/include/pdc_malloc.h b/src/utils/include/pdc_malloc.h deleted file mode 100644 index e8ea1941e..000000000 --- a/src/utils/include/pdc_malloc.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#ifndef PDC_MALLOC_H -#define PDC_MALLOC_H - -#include - -/***************************************/ -/* Library-private Function Prototypes */ -/***************************************/ -/** - * Create an object - * - * \param size [IN] Size of the struct to be malloced - */ -void *PDC_malloc(size_t size); - -/** - * Create an object - * - * \param size [IN] Size of the struct to be calloced - */ -void *PDC_calloc(size_t size); - -/** - * Create an object - * - * \param mem [IN] Starting address of memory - */ -void *PDC_free(void *mem); - -#define PDC_MALLOC(t) (t *)PDC_malloc(sizeof(t)) -#define PDC_CALLOC(t) (t *)PDC_calloc(sizeof(t)) - -#define PDC_FREE(t, obj) (t *)(intptr_t) PDC_free(obj) - -#endif /* PDC_MALLOC_H */ diff --git a/src/utils/include/pdc_private.h b/src/utils/include/pdc_private.h deleted file mode 100644 index b6ca3bcfd..000000000 --- a/src/utils/include/pdc_private.h +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#ifndef PDC_PRIVATE_H -#define PDC_PRIVATE_H - -#include "pdc_config.h" -#include "pdc_public.h" -#include -// #include /* gettimeofday() */ - -/****************************/ -/* Library Private Typedefs */ -/****************************/ -typedef enum { - UNKNOWN = 0, - SERVER_MEMORY = 1, - CLIENT_MEMORY = 2, - FLASH = 3, - DISK = 4, - FILESYSTEM = 5, - TAPE = 6 -} _pdc_loci_t; - -/* Query type */ -typedef enum { - PDC_Q_TYPE_DATA_ELEM, /* selects data elements */ - PDC_Q_TYPE_ATTR_VALUE, /* selects attribute values */ - PDC_Q_TYPE_ATTR_NAME, /* selects attributes */ - PDC_Q_TYPE_LINK_NAME, /* selects objects */ - PDC_Q_TYPE_MISC /* (for combine queries) selects misc objects */ -} _pdc_query_type_t; - -/* Query match conditions */ -typedef enum { - PDC_Q_MATCH_EQUAL, /* equal */ - PDC_Q_MATCH_NOT_EQUAL, /* not equal */ - PDC_Q_MATCH_LESS_THAN, /* less than */ - PDC_Q_MATCH_GREATER_THAN /* greater than */ -} _pdc_query_op_t; - -typedef enum { ROW_major, COL_major } _pdc_major_type_t; - -typedef enum { C_lang = 0, FORTRAN_lang, PYTHON_lang, JULIA_lang, N_LANGUAGES } _pdc_analysis_language_t; - -/***************************/ -/* Library Private Structs */ -/***************************/ -struct _pdc_class { - char * name; - pdcid_t local_id; -}; - -#ifdef __cplusplus -#define ATTRIBUTE(a) -#else /* __cplusplus */ -#if defined(HAVE_ATTRIBUTE) -#define ATTRIBUTE(a) __attribute__((a)) -#else -#define ATTRIBUTE(a) -#endif -#endif /* __cplusplus */ - -#ifdef __cplusplus -#define ATTR_UNUSED /*void*/ -#else /* __cplusplus */ -#if defined(HAVE_ATTRIBUTE) && !defined(__SUNPRO_C) -#define ATTR_UNUSED __attribute__((unused)) -#else -#define ATTR_UNUSED /*void*/ -#endif -#endif /* __cplusplus */ - -#define PDCmemset(X, C, Z) memset((void *)(X), C, Z) - -/* - * PDC Boolean type. - */ -#ifndef FALSE -#define FALSE 0 -#endif -#ifndef TRUE -#define TRUE 1 -#endif - -extern pbool_t err_occurred; - -/* - * PGOTO_DONE macro. The argument is the return value which is - * assigned to the `ret_value' variable. Control branches to - * the `done' label. - */ -#define PGOTO_DONE(ret_val) \ - do { \ - ret_value = ret_val; \ - goto done; \ - } while (0) - -#define PGOTO_DONE_VOID \ - do { \ - goto done; \ - } while (0) - -/* - * PGOTO_ERROR macro. The arguments are the return value and an - * error string. The return value is assigned to a variable `ret_value' and - * control branches to the `done' label. - */ -#define PGOTO_ERROR(ret_val, ...) \ - do { \ - fprintf(stderr, "Error in %s:%d\n", __FILE__, __LINE__); \ - fprintf(stderr, " # %s(): ", __func__); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - PGOTO_DONE(ret_val); \ - } while (0) - -#define PGOTO_ERROR_VOID(...) \ - do { \ - fprintf(stderr, "Error in %s:%d\n", __FILE__, __LINE__); \ - fprintf(stderr, " # %s(): ", __func__); \ - fprintf(stderr, "\n"); \ - PGOTO_DONE_VOID; \ - } while (0) - -/* Include a basic profiling interface */ -#ifdef ENABLE_PROFILING -#include "stack_ops.h" - -#define FUNC_ENTER(X) \ - do { \ - if (enableProfiling) \ - push(__func__, (X)); \ - } while (0) - -#define FUNC_LEAVE(ret_value) \ - do { \ - if (enableProfiling) \ - pop(); \ - return (ret_value); \ - } while (0) - -#define FUNC_LEAVE_VOID \ - do { \ - if (enableProfiling) \ - pop(); \ - return; \ - } while (0) - -#else -/* #define FUNC_ENTER(X) \ */ -/* do { \ */ -/* time_t now; \ */ -/* time(&now); \ */ -/* fprintf(stderr, "%ld enter %s\n", now, __func__); \ */ -/* } while (0) */ - -/* #define FUNC_LEAVE(ret_value) \ */ -/* do { \ */ -/* time_t now; \ */ -/* time(&now); \ */ -/* fprintf(stderr, "%ld leave %s\n", now, __func__); \ */ -/* return (ret_value); \ */ -/* } while (0) */ - -#define FUNC_ENTER(X) \ - do { \ - } while (0) - -#define FUNC_LEAVE(ret_value) \ - do { \ - return (ret_value); \ - } while (0) - -#define FUNC_LEAVE_VOID \ - do { \ - return; \ - } while (0) -#endif - -#endif /* PDC_PRIVATE_H */ diff --git a/src/utils/include/pdc_timing.h b/src/utils/include/pdc_timing.h deleted file mode 100644 index 1ea2f475b..000000000 --- a/src/utils/include/pdc_timing.h +++ /dev/null @@ -1,193 +0,0 @@ -#ifndef PDC_TIMING_H -#define PDC_TIMING_H - -#ifndef HOST_NAME_MAX -#if defined(__APPLE__) -#define HOST_NAME_MAX 255 -#else -#define HOST_NAME_MAX 64 -#endif /* __APPLE__ */ -#endif /* HOST_NAME_MAX */ - -#include "pdc_config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef PDC_TIMING -typedef struct pdc_timing { - double PDCbuf_obj_map_rpc; - double PDCbuf_obj_unmap_rpc; - - double PDCreg_obtain_lock_write_rpc; - double PDCreg_obtain_lock_read_rpc; - - double PDCreg_release_lock_write_rpc; - double PDCreg_release_lock_read_rpc; - - double PDCbuf_obj_map_rpc_wait; - double PDCbuf_obj_unmap_rpc_wait; - - double PDCreg_obtain_lock_write_rpc_wait; - double PDCreg_obtain_lock_read_rpc_wait; - double PDCreg_release_lock_write_rpc_wait; - double PDCreg_release_lock_read_rpc_wait; - - double PDCtransfer_request_start_write_rpc; - double PDCtransfer_request_wait_write_rpc; - double PDCtransfer_request_start_read_rpc; - double PDCtransfer_request_wait_read_rpc; - - double PDCtransfer_request_start_write_rpc_wait; - double PDCtransfer_request_start_read_rpc_wait; - double PDCtransfer_request_wait_write_rpc_wait; - double PDCtransfer_request_wait_read_rpc_wait; - - double PDCtransfer_request_start_all_write_rpc; - double PDCtransfer_request_start_all_read_rpc; - double PDCtransfer_request_wait_all_rpc; - - double PDCtransfer_request_start_all_write_rpc_wait; - double PDCtransfer_request_start_all_read_rpc_wait; - double PDCtransfer_request_wait_all_rpc_wait; - - double PDCtransfer_request_metadata_query_rpc; - - double PDCclient_obj_create_rpc; - double PDCclient_cont_create_rpc; - -} pdc_timing; - -pdc_timing pdc_timings; - -typedef struct pdc_server_timing { - double PDCbuf_obj_map_rpc; - double PDCbuf_obj_unmap_rpc; - - double PDCreg_obtain_lock_write_rpc; - double PDCreg_obtain_lock_read_rpc; - double PDCreg_release_lock_write_rpc; - double PDCreg_release_lock_read_rpc; - double PDCreg_release_lock_bulk_transfer_write_rpc; - double PDCreg_release_lock_bulk_transfer_read_rpc; - double PDCreg_release_lock_bulk_transfer_inner_write_rpc; - double PDCreg_release_lock_bulk_transfer_inner_read_rpc; - - double PDCreg_transfer_request_start_write_rpc; - double PDCreg_transfer_request_start_read_rpc; - double PDCreg_transfer_request_wait_write_rpc; - double PDCreg_transfer_request_wait_read_rpc; - double PDCreg_transfer_request_start_write_bulk_rpc; - double PDCreg_transfer_request_inner_write_bulk_rpc; - double PDCreg_transfer_request_start_read_bulk_rpc; - double PDCreg_transfer_request_inner_read_bulk_rpc; - - double PDCreg_transfer_request_start_all_write_rpc; - double PDCreg_transfer_request_start_all_read_rpc; - double PDCreg_transfer_request_start_all_write_bulk_rpc; - double PDCreg_transfer_request_start_all_read_bulk_rpc; - double PDCreg_transfer_request_inner_write_all_bulk_rpc; - double PDCreg_transfer_request_inner_read_all_bulk_rpc; - double PDCreg_transfer_request_wait_all_rpc; - double PDCreg_transfer_request_wait_all_bulk_rpc; - - double PDCdata_server_write_out; - double PDCdata_server_read_from; - double PDCcache_write; - double PDCcache_read; - double PDCcache_flush; - double PDCcache_clean; - double PDCdata_server_write_posix; - double PDCdata_server_read_posix; - - double PDCserver_obj_create_rpc; - double PDCserver_cont_create_rpc; - - double PDCserver_restart; - double PDCserver_checkpoint; - double PDCserver_start_total; -} pdc_server_timing; - -typedef struct pdc_timestamp { - double *start; - double *end; - size_t timestamp_max_size; - size_t timestamp_size; -} pdc_timestamp; - -pdc_server_timing *pdc_server_timings; -pdc_timestamp * pdc_buf_obj_map_timestamps; -pdc_timestamp * pdc_buf_obj_unmap_timestamps; - -pdc_timestamp *pdc_obtain_lock_write_timestamps; -pdc_timestamp *pdc_obtain_lock_read_timestamps; -pdc_timestamp *pdc_release_lock_write_timestamps; -pdc_timestamp *pdc_release_lock_read_timestamps; -pdc_timestamp *pdc_release_lock_bulk_transfer_write_timestamps; -pdc_timestamp *pdc_release_lock_bulk_transfer_inner_write_timestamps; -pdc_timestamp *pdc_release_lock_bulk_transfer_read_timestamps; -pdc_timestamp *pdc_release_lock_bulk_transfer_inner_read_timestamps; - -pdc_timestamp *pdc_transfer_request_start_write_timestamps; -pdc_timestamp *pdc_transfer_request_start_read_timestamps; -pdc_timestamp *pdc_transfer_request_wait_write_timestamps; -pdc_timestamp *pdc_transfer_request_wait_read_timestamps; -pdc_timestamp *pdc_transfer_request_start_write_bulk_timestamps; -pdc_timestamp *pdc_transfer_request_inner_write_bulk_timestamps; -pdc_timestamp *pdc_transfer_request_start_read_bulk_timestamps; -pdc_timestamp *pdc_transfer_request_inner_read_bulk_timestamps; - -pdc_timestamp *pdc_transfer_request_start_all_write_timestamps; -pdc_timestamp *pdc_transfer_request_start_all_read_timestamps; -pdc_timestamp *pdc_transfer_request_start_all_write_bulk_timestamps; -pdc_timestamp *pdc_transfer_request_start_all_read_bulk_timestamps; -pdc_timestamp *pdc_transfer_request_wait_all_timestamps; -pdc_timestamp *pdc_transfer_request_inner_write_all_bulk_timestamps; -pdc_timestamp *pdc_transfer_request_inner_read_all_bulk_timestamps; - -pdc_timestamp *pdc_client_buf_obj_map_timestamps; -pdc_timestamp *pdc_client_buf_obj_unmap_timestamps; -pdc_timestamp *pdc_client_obtain_lock_write_timestamps; -pdc_timestamp *pdc_client_obtain_lock_read_timestamps; -pdc_timestamp *pdc_client_release_lock_write_timestamps; -pdc_timestamp *pdc_client_release_lock_read_timestamps; - -pdc_timestamp *pdc_client_transfer_request_start_write_timestamps; -pdc_timestamp *pdc_client_transfer_request_start_read_timestamps; -pdc_timestamp *pdc_client_transfer_request_wait_write_timestamps; -pdc_timestamp *pdc_client_transfer_request_wait_read_timestamps; - -pdc_timestamp *pdc_client_transfer_request_start_all_write_timestamps; -pdc_timestamp *pdc_client_transfer_request_start_all_read_timestamps; -pdc_timestamp *pdc_client_transfer_request_wait_all_timestamps; - -pdc_timestamp *pdc_client_create_cont_timestamps; -pdc_timestamp *pdc_client_create_obj_timestamps; - -pdc_timestamp *pdc_client_transfer_request_metadata_query_timestamps; - -int PDC_timing_init(); -int PDC_timing_finalize(); -int PDC_timing_report(const char *prefix); -int PDC_server_timing_init(); -int pdc_timestamp_register(pdc_timestamp *timestamp, double start, double end); -int PDC_server_timing_report(); -#else -int PDC_timing_report(const char *prefix); -#endif - -#endif diff --git a/src/utils/pdc_interface.c b/src/utils/pdc_interface.c index 4cca6b0bc..218d6518e 100644 --- a/src/utils/pdc_interface.c +++ b/src/utils/pdc_interface.c @@ -25,6 +25,8 @@ #include "pdc_malloc.h" #include "pdc_id_pkg.h" #include "pdc_interface.h" +#include "pdc_cont_pkg.h" +#include "pdc_cont.h" #include #include diff --git a/src/utils/pdc_malloc.c b/src/utils/pdc_malloc.c deleted file mode 100644 index 40379d400..000000000 --- a/src/utils/pdc_malloc.c +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#include -#include -#include "pdc_malloc.h" -#include "pdc_private.h" - -void * -PDC_malloc(size_t size) -{ - void *ret_value; - - FUNC_ENTER(NULL); - - assert(size); - - if (size) - ret_value = malloc(size); - else - ret_value = NULL; - - FUNC_LEAVE(ret_value); -} - -void * -PDC_calloc(size_t size) -{ - void *ret_value; - - FUNC_ENTER(NULL); - - assert(size); - - if (size) - ret_value = calloc(1, size); - else - ret_value = NULL; - - FUNC_LEAVE(ret_value); -} - -void * -PDC_free(void *mem) -{ - void *ret_value = NULL; - - FUNC_ENTER(NULL); - - if (mem) { - free(mem); - } - - FUNC_LEAVE(ret_value); -} diff --git a/src/utils/pdc_timing.c b/src/utils/pdc_timing.c deleted file mode 100644 index 95826b56b..000000000 --- a/src/utils/pdc_timing.c +++ /dev/null @@ -1,537 +0,0 @@ -#include "pdc_timing.h" - -#ifdef PDC_TIMING -static double pdc_base_time; - -static int -pdc_timestamp_clean(pdc_timestamp *timestamp) -{ - if (timestamp->timestamp_size) { - free(timestamp->start); - } - return 0; -} - -static int -timestamp_log(FILE *stream, const char *header, pdc_timestamp *timestamp) -{ - size_t i; - double total = 0.0; - fprintf(stream, "%s", header); - for (i = 0; i < timestamp->timestamp_size; ++i) { - fprintf(stream, ",%4f-%4f", timestamp->start[i], timestamp->end[i]); - total += timestamp->end[i] - timestamp->start[i]; - } - fprintf(stream, "\n"); - - if (i > 0) - fprintf(stream, "%s_total, %f\n", header, total); - - return 0; -} - -int -PDC_timing_init() -{ - char hostname[HOST_NAME_MAX]; - int rank; - pdc_timestamp *ptr; - - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - gethostname(hostname, HOST_NAME_MAX); - if (!(rank % 31)) { - printf("client process rank %d, hostname = %s\n", rank, hostname); - } - MPI_Barrier(MPI_COMM_WORLD); - - memset(&pdc_timings, 0, sizeof(pdc_timing)); - - pdc_client_buf_obj_map_timestamps = calloc(16, sizeof(pdc_timestamp)); - ptr = pdc_client_buf_obj_map_timestamps + 1; - pdc_client_buf_obj_unmap_timestamps = ptr; - ptr++; - pdc_client_obtain_lock_write_timestamps = ptr; - ptr++; - pdc_client_obtain_lock_read_timestamps = ptr; - ptr++; - pdc_client_release_lock_write_timestamps = ptr; - ptr++; - pdc_client_release_lock_read_timestamps = ptr; - ptr++; - - pdc_client_transfer_request_start_write_timestamps = ptr; - ptr++; - pdc_client_transfer_request_start_read_timestamps = ptr; - ptr++; - pdc_client_transfer_request_wait_write_timestamps = ptr; - ptr++; - pdc_client_transfer_request_wait_read_timestamps = ptr; - ptr++; - - pdc_client_transfer_request_start_all_write_timestamps = ptr; - ptr++; - pdc_client_transfer_request_start_all_read_timestamps = ptr; - ptr++; - pdc_client_transfer_request_wait_all_timestamps = ptr; - ptr++; - - pdc_client_create_cont_timestamps = ptr; - ptr++; - pdc_client_create_obj_timestamps = ptr; - - ptr++; - pdc_client_transfer_request_metadata_query_timestamps = ptr; - - return 0; -} - -int -PDC_timing_finalize() -{ - pdc_timestamp_clean(pdc_client_buf_obj_map_timestamps); - pdc_timestamp_clean(pdc_client_buf_obj_unmap_timestamps); - - pdc_timestamp_clean(pdc_client_obtain_lock_write_timestamps); - pdc_timestamp_clean(pdc_client_obtain_lock_read_timestamps); - pdc_timestamp_clean(pdc_client_release_lock_write_timestamps); - pdc_timestamp_clean(pdc_client_release_lock_read_timestamps); - - pdc_timestamp_clean(pdc_client_transfer_request_start_write_timestamps); - pdc_timestamp_clean(pdc_client_transfer_request_start_read_timestamps); - pdc_timestamp_clean(pdc_client_transfer_request_wait_write_timestamps); - pdc_timestamp_clean(pdc_client_transfer_request_wait_read_timestamps); - pdc_timestamp_clean(pdc_client_create_cont_timestamps); - pdc_timestamp_clean(pdc_client_create_obj_timestamps); - pdc_timestamp_clean(pdc_client_transfer_request_start_all_write_timestamps); - pdc_timestamp_clean(pdc_client_transfer_request_start_all_read_timestamps); - pdc_timestamp_clean(pdc_client_transfer_request_wait_all_timestamps); - pdc_timestamp_clean(pdc_client_transfer_request_metadata_query_timestamps); - - free(pdc_client_buf_obj_map_timestamps); - return 0; -} - -int -PDC_timing_report(const char *prefix) -{ - pdc_timing max_timings; - int rank; - char filename[256], header[256]; - FILE * stream; - char hostname[HOST_NAME_MAX]; - time_t now; - - time(&now); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - gethostname(hostname, HOST_NAME_MAX); - if (!(rank % 32)) { - printf("client process rank %d, hostname = %s\n", rank, hostname); - } - MPI_Reduce(&pdc_timings, &max_timings, sizeof(pdc_timing) / sizeof(double), MPI_DOUBLE, MPI_MAX, 0, - MPI_COMM_WORLD); - if (rank == 0) { - printf("PDCbuf_obj_map_rpc = %lf, wait = %lf\n", max_timings.PDCbuf_obj_map_rpc, - max_timings.PDCbuf_obj_map_rpc_wait); - printf("PDCreg_obtain_lock_write_rpc = %lf, wait = %lf\n", max_timings.PDCreg_obtain_lock_write_rpc, - max_timings.PDCreg_obtain_lock_write_rpc_wait); - printf("PDCreg_obtain_lock_read_rpc = %lf, wait = %lf\n", max_timings.PDCreg_obtain_lock_read_rpc, - max_timings.PDCreg_obtain_lock_read_rpc_wait); - - printf("PDCreg_release_lock_write_rpc = %lf, wait = %lf\n", max_timings.PDCreg_release_lock_write_rpc, - max_timings.PDCreg_release_lock_write_rpc_wait); - printf("PDCreg_release_lock_read_rpc = %lf, wait = %lf\n", max_timings.PDCreg_release_lock_read_rpc, - max_timings.PDCreg_release_lock_read_rpc_wait); - printf("PDCbuf_obj_unmap_rpc = %lf, wait = %lf\n", max_timings.PDCbuf_obj_unmap_rpc, - max_timings.PDCbuf_obj_unmap_rpc_wait); - - printf("PDCtransfer_request_start_write = %lf, wait = %lf\n", - max_timings.PDCtransfer_request_start_write_rpc, - max_timings.PDCtransfer_request_start_write_rpc_wait); - printf("PDCtransfer_request_start_read = %lf, wait = %lf\n", - max_timings.PDCtransfer_request_start_read_rpc, - max_timings.PDCtransfer_request_start_read_rpc_wait); - printf("PDCtransfer_request_wait_write = %lf, wait = %lf\n", - max_timings.PDCtransfer_request_wait_write_rpc, - max_timings.PDCtransfer_request_wait_write_rpc_wait); - printf("PDCtransfer_request_wait_read = %lf, wait = %lf\n", - max_timings.PDCtransfer_request_wait_read_rpc, - max_timings.PDCtransfer_request_wait_read_rpc_wait); - printf("PDCtransfer_request_start_all_write = %lf, wait = %lf\n", - max_timings.PDCtransfer_request_start_all_write_rpc, - max_timings.PDCtransfer_request_start_all_write_rpc_wait); - printf("PDCtransfer_request_start_all_read = %lf, wait = %lf\n", - max_timings.PDCtransfer_request_start_all_read_rpc, - max_timings.PDCtransfer_request_start_all_read_rpc_wait); - printf("PDCtransfer_request_wait_write = %lf, wait = %lf\n", - max_timings.PDCtransfer_request_wait_all_rpc, - max_timings.PDCtransfer_request_wait_all_rpc_wait); - } - - sprintf(filename, "pdc_client_log_rank_%d.csv", rank); - stream = fopen(filename, "r"); - if (stream) { - fclose(stream); - stream = fopen(filename, "a"); - } - else { - stream = fopen(filename, "w"); - } - - fprintf(stream, "%s", ctime(&now)); - - sprintf(header, "buf_obj_map_%s", prefix); - timestamp_log(stream, header, pdc_client_buf_obj_map_timestamps); - sprintf(header, "buf_obj_unmap_%s", prefix); - timestamp_log(stream, header, pdc_client_buf_obj_unmap_timestamps); - - sprintf(header, "obtain_lock_write_%s", prefix); - timestamp_log(stream, header, pdc_client_obtain_lock_write_timestamps); - sprintf(header, "obtain_lock_read_%s", prefix); - timestamp_log(stream, header, pdc_client_obtain_lock_read_timestamps); - - sprintf(header, "release_lock_write_%s", prefix); - timestamp_log(stream, header, pdc_client_release_lock_write_timestamps); - sprintf(header, "release_lock_read_%s", prefix); - timestamp_log(stream, header, pdc_client_release_lock_read_timestamps); - - sprintf(header, "transfer_request_start_write_%s", prefix); - timestamp_log(stream, header, pdc_client_transfer_request_start_write_timestamps); - - sprintf(header, "transfer_request_start_read_%s", prefix); - timestamp_log(stream, header, pdc_client_transfer_request_start_read_timestamps); - - sprintf(header, "transfer_request_wait_write_%s", prefix); - timestamp_log(stream, header, pdc_client_transfer_request_wait_write_timestamps); - - sprintf(header, "transfer_request_wait_read_%s", prefix); - timestamp_log(stream, header, pdc_client_transfer_request_wait_read_timestamps); - - sprintf(header, "transfer_request_start_all_write_%s", prefix); - timestamp_log(stream, header, pdc_client_transfer_request_start_all_write_timestamps); - - sprintf(header, "transfer_request_start_all_read_%s", prefix); - timestamp_log(stream, header, pdc_client_transfer_request_start_all_read_timestamps); - - sprintf(header, "transfer_request_wait_all_%s", prefix); - timestamp_log(stream, header, pdc_client_transfer_request_wait_all_timestamps); - - sprintf(header, "create_cont"); - timestamp_log(stream, header, pdc_client_create_cont_timestamps); - - sprintf(header, "create_obj"); - timestamp_log(stream, header, pdc_client_create_obj_timestamps); - - fprintf(stream, "\n"); - fclose(stream); - - pdc_client_buf_obj_map_timestamps->timestamp_size = 0; - pdc_client_buf_obj_unmap_timestamps->timestamp_size = 0; - - pdc_client_obtain_lock_write_timestamps->timestamp_size = 0; - pdc_client_obtain_lock_read_timestamps->timestamp_size = 0; - pdc_client_release_lock_write_timestamps->timestamp_size = 0; - pdc_client_release_lock_read_timestamps->timestamp_size = 0; - - pdc_client_transfer_request_start_write_timestamps->timestamp_size = 0; - pdc_client_transfer_request_start_read_timestamps->timestamp_size = 0; - pdc_client_transfer_request_wait_write_timestamps->timestamp_size = 0; - pdc_client_transfer_request_wait_read_timestamps->timestamp_size = 0; - - pdc_client_transfer_request_start_all_write_timestamps->timestamp_size = 0; - pdc_client_transfer_request_start_all_read_timestamps->timestamp_size = 0; - pdc_client_transfer_request_wait_all_timestamps->timestamp_size = 0; - - pdc_client_create_cont_timestamps->timestamp_size = 0; - pdc_client_create_obj_timestamps->timestamp_size = 0; - - pdc_client_transfer_request_metadata_query_timestamps->timestamp_size = 0; - - memset(&pdc_timings, 0, sizeof(pdc_timings)); - - return 0; -} - -int -PDC_server_timing_init() -{ - char hostname[HOST_NAME_MAX]; - int rank; - - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - gethostname(hostname, HOST_NAME_MAX); - - printf("server process rank %d, hostname = %s\n", rank, hostname); - /* - printf("rank = %d, hostname = %s, PDCbuf_obj_map_rpc = %lf, PDCreg_obtain_lock_rpc = %lf, " - "PDCreg_release_lock_write_rpc = " - "%lf, PDCreg_release_lock_read_rpc = %lf, PDCbuf_obj_unmap_rpc = %lf, " - "region_release_bulk_transfer_cb = %lf\n", - rank, hostname, server_timings->PDCbuf_obj_map_rpc, server_timings->PDCreg_obtain_lock_rpc, - server_timings->PDCreg_release_lock_write_rpc, server_timings->PDCreg_release_lock_read_rpc, - server_timings->PDCbuf_obj_unmap_rpc, server_timings->PDCreg_release_lock_bulk_transfer_rpc); - */ - MPI_Barrier(MPI_COMM_WORLD); - - pdc_server_timings = calloc(1, sizeof(pdc_server_timing)); - pdc_timestamp *ptr = calloc(25, sizeof(pdc_timestamp)); - pdc_buf_obj_map_timestamps = ptr; - ptr++; - pdc_buf_obj_unmap_timestamps = ptr; - ptr++; - pdc_obtain_lock_write_timestamps = ptr; - ptr++; - pdc_obtain_lock_read_timestamps = ptr; - ptr++; - pdc_release_lock_write_timestamps = ptr; - ptr++; - pdc_release_lock_read_timestamps = ptr; - ptr++; - pdc_release_lock_bulk_transfer_write_timestamps = ptr; - ptr++; - pdc_release_lock_bulk_transfer_read_timestamps = ptr; - ptr++; - pdc_release_lock_bulk_transfer_inner_write_timestamps = ptr; - ptr++; - pdc_release_lock_bulk_transfer_inner_read_timestamps = ptr; - ptr++; - - pdc_transfer_request_start_write_timestamps = ptr; - ptr++; - pdc_transfer_request_start_read_timestamps = ptr; - ptr++; - pdc_transfer_request_wait_write_timestamps = ptr; - ptr++; - pdc_transfer_request_wait_read_timestamps = ptr; - ptr++; - pdc_transfer_request_start_write_bulk_timestamps = ptr; - ptr++; - pdc_transfer_request_start_read_bulk_timestamps = ptr; - ptr++; - pdc_transfer_request_inner_write_bulk_timestamps = ptr; - ptr++; - pdc_transfer_request_inner_read_bulk_timestamps = ptr; - ptr++; - - pdc_transfer_request_start_all_write_timestamps = ptr; - ptr++; - pdc_transfer_request_start_all_read_timestamps = ptr; - ptr++; - pdc_transfer_request_wait_all_timestamps = ptr; - ptr++; - pdc_transfer_request_start_all_write_bulk_timestamps = ptr; - ptr++; - pdc_transfer_request_start_all_read_bulk_timestamps = ptr; - ptr++; - pdc_transfer_request_inner_write_all_bulk_timestamps = ptr; - ptr++; - pdc_transfer_request_inner_read_all_bulk_timestamps = ptr; - ptr++; - - // 25 timestamps - - pdc_base_time = MPI_Wtime(); - return 0; -} - -int -pdc_timestamp_register(pdc_timestamp *timestamp, double start, double end) -{ - double *temp; - - if (timestamp->timestamp_max_size == 0) { - timestamp->timestamp_max_size = 256; - timestamp->start = (double *)malloc(sizeof(double) * timestamp->timestamp_max_size * 2); - timestamp->end = timestamp->start + timestamp->timestamp_max_size; - timestamp->timestamp_size = 0; - } - else if (timestamp->timestamp_size == timestamp->timestamp_max_size) { - temp = (double *)malloc(sizeof(double) * timestamp->timestamp_max_size * 4); - memcpy(temp, timestamp->start, sizeof(double) * timestamp->timestamp_max_size); - memcpy(temp + timestamp->timestamp_max_size * 2, timestamp->end, - sizeof(double) * timestamp->timestamp_max_size); - timestamp->start = temp; - timestamp->end = temp + timestamp->timestamp_max_size * 2; - timestamp->timestamp_max_size *= 2; - } - timestamp->start[timestamp->timestamp_size] = start; - timestamp->end[timestamp->timestamp_size] = end; - timestamp->timestamp_size++; - return 0; -} - -int -PDC_server_timing_report() -{ - pdc_server_timing max_timings; - int rank; - char filename[256]; - FILE * stream; - - // char hostname[HOST_NAME_MAX]; - time_t now; - - time(&now); - - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Reduce(pdc_server_timings, &max_timings, sizeof(pdc_server_timing) / sizeof(double), MPI_DOUBLE, - MPI_MAX, 0, MPI_COMM_WORLD); - sprintf(filename, "pdc_server_log_rank_%d.csv", rank); - - stream = fopen(filename, "w"); - - fprintf(stream, "%s", ctime(&now)); - timestamp_log(stream, "buf_obj_map", pdc_buf_obj_map_timestamps); - timestamp_log(stream, "buf_obj_unmap", pdc_buf_obj_unmap_timestamps); - - timestamp_log(stream, "obtain_lock_write", pdc_obtain_lock_write_timestamps); - timestamp_log(stream, "obtain_lock_read", pdc_obtain_lock_read_timestamps); - timestamp_log(stream, "release_lock_write", pdc_release_lock_write_timestamps); - timestamp_log(stream, "release_lock_read", pdc_release_lock_read_timestamps); - timestamp_log(stream, "release_lock_bulk_transfer_write", - pdc_release_lock_bulk_transfer_write_timestamps); - timestamp_log(stream, "release_lock_bulk_transfer_read", pdc_release_lock_bulk_transfer_read_timestamps); - timestamp_log(stream, "release_lock_bulk_transfer_inner_write", - pdc_release_lock_bulk_transfer_inner_write_timestamps); - timestamp_log(stream, "release_lock_bulk_transfer_inner_read", - pdc_release_lock_bulk_transfer_inner_read_timestamps); - - timestamp_log(stream, "transfer_request_start_write", pdc_transfer_request_start_write_timestamps); - timestamp_log(stream, "transfer_request_wait_write", pdc_transfer_request_wait_write_timestamps); - timestamp_log(stream, "transfer_request_start_write_bulk", - pdc_transfer_request_start_write_bulk_timestamps); - timestamp_log(stream, "transfer_request_inner_write_bulk", - pdc_transfer_request_inner_write_bulk_timestamps); - timestamp_log(stream, "transfer_request_start_read", pdc_transfer_request_start_read_timestamps); - timestamp_log(stream, "transfer_request_wait_read", pdc_transfer_request_wait_read_timestamps); - timestamp_log(stream, "transfer_request_start_read_bulk", - pdc_transfer_request_start_read_bulk_timestamps); - timestamp_log(stream, "transfer_request_inner_read_bulk", - pdc_transfer_request_inner_read_bulk_timestamps); - - timestamp_log(stream, "transfer_request_start_all_write", - pdc_transfer_request_start_all_write_timestamps); - timestamp_log(stream, "transfer_request_start_all_write_bulk", - pdc_transfer_request_start_all_write_bulk_timestamps); - timestamp_log(stream, "transfer_request_start_all_read", pdc_transfer_request_start_all_read_timestamps); - timestamp_log(stream, "transfer_request_start_all_read_bulk", - pdc_transfer_request_start_all_read_bulk_timestamps); - timestamp_log(stream, "transfer_request_inner_write_all_bulk", - pdc_transfer_request_inner_write_all_bulk_timestamps); - timestamp_log(stream, "transfer_request_inner_read_all_bulk", - pdc_transfer_request_inner_read_all_bulk_timestamps); - timestamp_log(stream, "transfer_request_wait_all", pdc_transfer_request_wait_all_timestamps); - - /* timestamp_log(stream, "create_obj", create_obj_timestamps); */ - /* timestamp_log(stream, "create_cont", create_cont_timestamps); */ - fclose(stream); - - sprintf(filename, "pdc_server_timings_%d.csv", rank); - stream = fopen(filename, "w"); - fprintf(stream, "%s", ctime(&now)); - fprintf(stream, "PDCbuf_obj_map_rpc, %lf\n", pdc_server_timings->PDCbuf_obj_map_rpc); - fprintf(stream, "PDCreg_obtain_lock_write_rpc, %lf\n", pdc_server_timings->PDCreg_obtain_lock_write_rpc); - fprintf(stream, "PDCreg_obtain_lock_read_rpc, %lf\n", pdc_server_timings->PDCreg_obtain_lock_read_rpc); - fprintf(stream, "PDCreg_release_lock_write_rpc, %lf\n", - pdc_server_timings->PDCreg_release_lock_write_rpc); - fprintf(stream, "PDCreg_release_lock_read_rpc, %lf\n", pdc_server_timings->PDCreg_release_lock_read_rpc); - fprintf(stream, "PDCbuf_obj_unmap_rpc, %lf\n", pdc_server_timings->PDCbuf_obj_unmap_rpc); - fprintf(stream, "PDCreg_release_lock_bulk_transfer_write_rpc, %lf\n", - pdc_server_timings->PDCreg_release_lock_bulk_transfer_write_rpc); - fprintf(stream, "PDCreg_release_lock_bulk_transfer_read_rpc, %lf\n", - pdc_server_timings->PDCreg_release_lock_bulk_transfer_read_rpc); - fprintf(stream, "PDCreg_release_lock_bulk_transfer_inner_write_rpc, %lf\n", - pdc_server_timings->PDCreg_release_lock_bulk_transfer_inner_write_rpc); - fprintf(stream, "PDCreg_release_lock_bulk_transfer_inner_read_rpc, %lf\n", - pdc_server_timings->PDCreg_release_lock_bulk_transfer_inner_read_rpc); - fprintf(stream, "PDCregion_transfer_start_write_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_start_write_rpc); - fprintf(stream, "PDCregion_transfer_wait_write_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_wait_write_rpc); - fprintf(stream, "PDCregion_transfer_start_write_bulk_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_start_write_bulk_rpc); - fprintf(stream, "PDCregion_transfer_request_inner_write_bulk_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_inner_write_bulk_rpc); - fprintf(stream, "PDCregion_transfer_start_read_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_start_read_rpc); - fprintf(stream, "PDCregion_transfer_wait_read_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_wait_read_rpc); - fprintf(stream, "PDCregion_transfer_start_read_bulk_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_start_read_bulk_rpc); - fprintf(stream, "PDCregion_transfer_request_inner_read_bulk_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_inner_read_bulk_rpc); - - fprintf(stream, "PDCregion_transfer_start_write_all_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_start_all_write_rpc); - fprintf(stream, "PDCregion_transfer_request_inner_write_all_bulk_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_inner_write_all_bulk_rpc); - fprintf(stream, "PDCregion_transfer_start_all_read_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_start_all_read_rpc); - fprintf(stream, "PDCregion_transfer_request_inner_read_all_bulk_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_inner_read_all_bulk_rpc); - fprintf(stream, "PDCregion_transfer_wait_all_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_wait_all_rpc); - fprintf(stream, "PDCregion_transfer_wait_all_bulk_rpc, %lf\n", - pdc_server_timings->PDCreg_transfer_request_wait_all_bulk_rpc); - - fprintf(stream, "PDCserver_obj_create_rpc, %lf\n", pdc_server_timings->PDCserver_obj_create_rpc); - fprintf(stream, "PDCserver_cont_create_rpc, %lf\n", pdc_server_timings->PDCserver_cont_create_rpc); - - fprintf(stream, "PDCdata_server_write_out, %lf\n", pdc_server_timings->PDCdata_server_write_out); - fprintf(stream, "PDCdata_server_read_from, %lf\n", pdc_server_timings->PDCdata_server_read_from); - fprintf(stream, "PDCcache_write, %lf\n", pdc_server_timings->PDCcache_write); - fprintf(stream, "PDCcache_read, %lf\n", pdc_server_timings->PDCcache_read); - fprintf(stream, "PDCcache_flush, %lf\n", pdc_server_timings->PDCcache_flush); - fprintf(stream, "PDCcache_clean, %lf\n", pdc_server_timings->PDCcache_clean); - fprintf(stream, "PDCdata_server_write_posix, %lf\n", pdc_server_timings->PDCdata_server_write_posix); - fprintf(stream, "PDCdata_server_read_posix, %lf\n", pdc_server_timings->PDCdata_server_read_posix); - - fprintf(stream, "PDCserver_restart, %lf\n", pdc_server_timings->PDCserver_restart); - fprintf(stream, "PDCserver_checkpoint, %lf\n", pdc_server_timings->PDCserver_checkpoint); - fprintf(stream, "PDCstart_server_total, %lf\n", pdc_server_timings->PDCserver_start_total); - - fclose(stream); - - free(pdc_server_timings); - pdc_timestamp_clean(pdc_buf_obj_map_timestamps); - pdc_timestamp_clean(pdc_buf_obj_unmap_timestamps); - - pdc_timestamp_clean(pdc_obtain_lock_write_timestamps); - pdc_timestamp_clean(pdc_obtain_lock_read_timestamps); - pdc_timestamp_clean(pdc_release_lock_write_timestamps); - pdc_timestamp_clean(pdc_release_lock_read_timestamps); - pdc_timestamp_clean(pdc_release_lock_bulk_transfer_write_timestamps); - pdc_timestamp_clean(pdc_release_lock_bulk_transfer_read_timestamps); - pdc_timestamp_clean(pdc_release_lock_bulk_transfer_inner_write_timestamps); - pdc_timestamp_clean(pdc_release_lock_bulk_transfer_inner_read_timestamps); - - pdc_timestamp_clean(pdc_transfer_request_start_write_timestamps); - pdc_timestamp_clean(pdc_transfer_request_start_read_timestamps); - pdc_timestamp_clean(pdc_transfer_request_wait_write_timestamps); - pdc_timestamp_clean(pdc_transfer_request_wait_read_timestamps); - pdc_timestamp_clean(pdc_transfer_request_start_write_bulk_timestamps); - pdc_timestamp_clean(pdc_transfer_request_start_read_bulk_timestamps); - pdc_timestamp_clean(pdc_transfer_request_inner_write_bulk_timestamps); - pdc_timestamp_clean(pdc_transfer_request_inner_read_bulk_timestamps); - - pdc_timestamp_clean(pdc_transfer_request_start_all_write_timestamps); - pdc_timestamp_clean(pdc_transfer_request_start_all_read_timestamps); - pdc_timestamp_clean(pdc_transfer_request_start_all_write_bulk_timestamps); - pdc_timestamp_clean(pdc_transfer_request_start_all_read_bulk_timestamps); - pdc_timestamp_clean(pdc_transfer_request_wait_all_timestamps); - pdc_timestamp_clean(pdc_transfer_request_inner_write_all_bulk_timestamps); - pdc_timestamp_clean(pdc_transfer_request_inner_read_all_bulk_timestamps); - - /* pdc_timestamp_clean(pdc_create_obj_timestamps); */ - /* pdc_timestamp_clean(pdc_create_cont_timestamps); */ - - free(pdc_buf_obj_map_timestamps); - return 0; -} - -#else -int -PDC_timing_report(const char *prefix __attribute__((unused))) -{ - return 0; -} -#endif diff --git a/tools/pdc_export.c b/tools/pdc_export.c index a67f444e3..9c0b572b2 100644 --- a/tools/pdc_export.c +++ b/tools/pdc_export.c @@ -7,6 +7,7 @@ #include #include #include "hdf5.h" +#include "pdc_generic.h" // #define ENABLE_MPI 1 @@ -240,45 +241,11 @@ get_data_type(int data_type) if (data_type == -1) { return "PDC_UNKNOWN"; } - else if (data_type == 0) { - return "PDC_INT"; - } - else if (data_type == 1) { - return "PDC_FLOAT"; - } - else if (data_type == 2) { - return "PDC_DOUBLE"; - } - else if (data_type == 3) { - return "PDC_CHAR"; - } - else if (data_type == 4) { - return "PDC_COMPOUND"; - } - else if (data_type == 5) { - return "PDC_ENUM"; - } - else if (data_type == 6) { - return "PDC_ARRAY"; - } - else if (data_type == 7) { - return "PDC_UINT"; - } - else if (data_type == 8) { - return "PDC_INT64"; - } - else if (data_type == 9) { - return "PDC_UINT64"; - } - else if (data_type == 10) { - return "PDC_INT16"; - } - else if (data_type == 11) { - return "PDC_INT16"; - } - else { + char *result = get_enum_name_by_dtype(data_type); + if (result == NULL) { return "NULL"; } + return result; } char * diff --git a/tools/pdc_import.c b/tools/pdc_import.c index f51f587c8..adf8cd46e 100644 --- a/tools/pdc_import.c +++ b/tools/pdc_import.c @@ -711,9 +711,10 @@ do_attr(hid_t aid, pdcid_t obj_id) char buf[MAX_NAME] = {0}; char read_buf[TAG_LEN_MAX] = {0}; // pdc_kvtag_t kvtag1; - char * tag_name; - void * tag_value; - size_t tag_size; + char * tag_name; + void * tag_value; + pdc_var_type_t value_type; + size_t tag_size; /* * Get the name of the attribute. @@ -735,7 +736,7 @@ do_attr(hid_t aid, pdcid_t obj_id) else { tag_size = H5Tget_size(atype); } - PDCobj_put_tag(obj_id, tag_name, tag_value, tag_size); + PDCobj_put_tag(obj_id, tag_name, tag_value, value_type, tag_size); /* * Get attribute information: dataspace, data type diff --git a/tools/pdc_ls.c b/tools/pdc_ls.c index 9d3db32fe..5963b33e2 100644 --- a/tools/pdc_ls.c +++ b/tools/pdc_ls.c @@ -201,45 +201,11 @@ get_data_type(int data_type) if (data_type == -1) { return "PDC_UNKNOWN"; } - else if (data_type == 0) { - return "PDC_INT"; - } - else if (data_type == 1) { - return "PDC_FLOAT"; - } - else if (data_type == 2) { - return "PDC_DOUBLE"; - } - else if (data_type == 3) { - return "PDC_CHAR"; - } - else if (data_type == 4) { - return "PDC_COMPOUND"; - } - else if (data_type == 5) { - return "PDC_ENUM"; - } - else if (data_type == 6) { - return "PDC_ARRAY"; - } - else if (data_type == 7) { - return "PDC_UINT"; - } - else if (data_type == 8) { - return "PDC_INT64"; - } - else if (data_type == 9) { - return "PDC_UINT64"; - } - else if (data_type == 10) { - return "PDC_INT16"; - } - else if (data_type == 11) { - return "PDC_INT16"; - } - else { + char *result = get_enum_name_by_dtype(data_type); + if (result == NULL) { return "NULL"; } + return result; } char * From 9fd90c29fd7aa61bdbc06e9ad6744b40346d9cae Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Mon, 19 Jun 2023 18:38:26 -0400 Subject: [PATCH 194/216] Feature/metadata type (#3) * upate metadata type system * update serde framework to coupe with the new data type system * replace unnecessary data types * adding type for pdc_kvtag_t, all occurances are fixed * update new commons CMake for publishing commons * commons compilation passed * compiled * remove unnecessary header files from installation * resolve conflict * add important files * clang formatting * update cmake --- src/api/CMakeLists.txt | 4 +- src/api/pdc_client_connect.c | 6 +- src/api/pdc_obj/include/pdc_cont.h | 6 +- src/commons/CMakeLists.txt | 27 +- src/commons/include/pdc_public.h | 62 ++ src/commons/profiling/CMakeLists.txt | 127 ++++ src/commons/profiling/include/pdc_hashtab.h | 198 +++++++ src/commons/profiling/include/pdc_stack_ops.h | 70 +++ src/commons/profiling/pdc_hashtab.c | 540 ++++++++++++++++++ src/commons/profiling/pdc_stack_ops.c | 264 +++++++++ src/commons/serde/include/pdc_serde.h | 21 +- src/commons/serde/pdc_serde.c | 22 +- src/commons/utils/include/pdc_id_pkg.h | 57 ++ src/commons/utils/include/pdc_linkedlist.h | 120 ++++ src/commons/utils/include/pdc_malloc.h | 59 ++ src/commons/utils/include/pdc_private.h | 202 +++++++ src/commons/utils/include/pdc_timing.h | 193 +++++++ src/commons/utils/pdc_malloc.c | 76 +++ src/commons/utils/pdc_timing.c | 537 +++++++++++++++++ src/server/pdc_client_server_common.c | 8 +- src/tests/cont_tags.c | 4 +- src/tests/kvtag_add_get_benchmark.c | 7 +- src/tests/kvtag_add_get_scale.c | 3 +- src/tests/obj_tags.c | 6 +- src/tests/read_obj_shared.c | 4 +- src/tests/vpicio_mts.c | 2 +- 26 files changed, 2559 insertions(+), 66 deletions(-) create mode 100644 src/commons/include/pdc_public.h create mode 100644 src/commons/profiling/CMakeLists.txt create mode 100644 src/commons/profiling/include/pdc_hashtab.h create mode 100644 src/commons/profiling/include/pdc_stack_ops.h create mode 100644 src/commons/profiling/pdc_hashtab.c create mode 100644 src/commons/profiling/pdc_stack_ops.c create mode 100644 src/commons/utils/include/pdc_id_pkg.h create mode 100644 src/commons/utils/include/pdc_linkedlist.h create mode 100644 src/commons/utils/include/pdc_malloc.h create mode 100644 src/commons/utils/include/pdc_private.h create mode 100644 src/commons/utils/include/pdc_timing.h create mode 100644 src/commons/utils/pdc_malloc.c create mode 100644 src/commons/utils/pdc_timing.c diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index 7ef5ec186..1ba4b8389 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -142,7 +142,6 @@ install( #----------------------------------------------------------------------------- set(PDC_HEADERS ${PDC_SOURCE_DIR}/src/api/include/pdc.h - ${PDC_SOURCE_DIR}/src/api/include/pdc_public.h ${PDC_SOURCE_DIR}/src/api/pdc_analysis/include/pdc_analysis.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_cont.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_mpi.h @@ -152,7 +151,6 @@ set(PDC_HEADERS ${PDC_SOURCE_DIR}/src/api/pdc_query/include/pdc_query.h ${PDC_SOURCE_DIR}/src/api/pdc_region/include/pdc_region.h ${PDC_SOURCE_DIR}/src/api/pdc_transform/include/pdc_transform.h - ${PDC_SOURCE_DIR}/src/utils/include/pdc_interface.h ${PROJECT_BINARY_DIR}/pdc_config_sys.h ${PROJECT_BINARY_DIR}/pdc_config.h ) @@ -232,4 +230,4 @@ set(PDC_INCLUDES_INSTALL_TIME ${PDC_INSTALL_INCLUDE_DIR} ${PDC_EXT_INCLUDE_DEPENDENCIES} PARENT_SCOPE -) +) \ No newline at end of file diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index b6b969f8a..fff47f027 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -7581,7 +7581,8 @@ PDCcont_get_objids(pdcid_t cont_id ATTRIBUTE(unused), int *nobj ATTRIBUTE(unused } perr_t -PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, psize_t value_size) +PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, + psize_t value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t kvtag; @@ -7604,7 +7605,8 @@ PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t } perr_t -PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, psize_t *value_size) +PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t *kvtag = NULL; diff --git a/src/api/pdc_obj/include/pdc_cont.h b/src/api/pdc_obj/include/pdc_cont.h index 3a6180b65..33e924c14 100644 --- a/src/api/pdc_obj/include/pdc_cont.h +++ b/src/api/pdc_obj/include/pdc_cont.h @@ -191,7 +191,8 @@ perr_t PDCcont_del(pdcid_t cont_id); * * \return Non-negative on success/Negative on failure */ -perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, psize_t value_size); +perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, + psize_t value_size); /** * *********** @@ -203,7 +204,8 @@ perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var * * \return Non-negative on success/Negative on failure */ -perr_t PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, psize_t *value_size); +perr_t PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size); /** * Deleta a tag from a container diff --git a/src/commons/CMakeLists.txt b/src/commons/CMakeLists.txt index f29c25a7c..9f8abf690 100644 --- a/src/commons/CMakeLists.txt +++ b/src/commons/CMakeLists.txt @@ -21,27 +21,11 @@ endif() # Mercury find_package(MERCURY REQUIRED) if(MERCURY_FOUND) - message(STATUS "mercury dir ${MERCURY_DIR}") - if(DEFINED MERCURY_DIR AND NOT "${MERCURY_DIR}" STREQUAL "") - # If MERCURY_DIR contains "share/" (or "share" is at the end), truncate it along with everything following it - string(REGEX REPLACE "/share.*" "" trimmed_mercury_dir ${MERCURY_DIR}) - # Check if the trimmed_mercury_dir ends with "/mercury" - string(REGEX MATCH ".*/mercury$" is_mercury_home ${trimmed_mercury_dir}) - # If trimmed_mercury_dir ends with "/mercury", consider it as the home directory - if(is_mercury_home) - set(MERCURY_HOME ${trimmed_mercury_dir}) - else() - # If not ending with "mercury", it's likely an error - message(FATAL_ERROR "Cannot determine MERCURY_HOME from MERCURY_DIR.") - endif() - - message("Mercury home is set to ${MERCURY_HOME}") - - set(MERCURY_INCLUDE_DIR ${MERCURY_HOME}/include) - set(MERCURY_LIBRARY_DIR ${MERCURY_HOME}/lib) - endif() - message(STATUS "mercury include dir ${MERCURY_INCLUDE_DIR}") - message(STATUS "mercury lib dir ${MERCURY_LIBRARY_DIR}") + message(STATUS "mercury dir = ${MERCURY_DIR}") + find_path(MERCURY_INCLUDE_DIR mercury.h HINTS ${MERCURY_DIR}) + find_library(MERCURY_LIBRARY mercury HINTS ${MERCURY_DIR}) + message(STATUS "mercury include dir = ${MERCURY_INCLUDE_DIR}") + message(STATUS "mercury lib = ${MERCURY_LIBRARY}") set(PDC_EXT_INCLUDE_DEPENDENCIES ${MERCURY_INCLUDE_DIR} ${PDC_EXT_INCLUDE_DEPENDENCIES} ) @@ -140,7 +124,6 @@ set(PUBLIC_HEADER_DIR_LIST ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/generic/include ${CMAKE_CURRENT_SOURCE_DIR}/profiling/include - ${CMAKE_CURRENT_SOURCE_DIR}/utils/include ) foreach(_header_dir ${PUBLIC_HEADER_DIR_LIST}) diff --git a/src/commons/include/pdc_public.h b/src/commons/include/pdc_public.h new file mode 100644 index 000000000..6c135f657 --- /dev/null +++ b/src/commons/include/pdc_public.h @@ -0,0 +1,62 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#ifndef PDC_PUBLIC_H +#define PDC_PUBLIC_H + +#include +#include +#include +#include "pdc_generic.h" + +/*******************/ +/* Public Typedefs */ +/*******************/ +typedef int perr_t; +typedef uint64_t pdcid_t; +typedef unsigned long long psize_t; +typedef bool pbool_t; + +typedef int PDC_int_t; +typedef float PDC_float_t; +typedef double PDC_double_t; + +typedef pdc_c_var_type_t pdc_var_type_t; + +typedef enum { PDC_PERSIST, PDC_TRANSIENT } pdc_lifetime_t; + +typedef enum { PDC_SERVER_DEFAULT = 0, PDC_SERVER_PER_CLIENT = 1 } pdc_server_selection_t; + +typedef struct pdc_histogram_t { //????????? + pdc_var_type_t dtype; + int nbin; + double incr; + double * range; + uint64_t * bin; +} pdc_histogram_t; + +#define SUCCEED 0 +#define FAIL (-1) + +#endif /* PDC_PUBLIC_H */ diff --git a/src/commons/profiling/CMakeLists.txt b/src/commons/profiling/CMakeLists.txt new file mode 100644 index 000000000..05e2c90b9 --- /dev/null +++ b/src/commons/profiling/CMakeLists.txt @@ -0,0 +1,127 @@ +#------------------------------------------------------------------------------ +# Include source and build directories +#------------------------------------------------------------------------------ +set(PROFILING_INCLUDE_DIRS + ${PDC_COMMON_INCLUDE_DIRS} + ${PDC_INCLUDES_BUILD_TIME} + ${PROJECT_SOURCE_DIR} + ${PROJECT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${PDC_SOURCE_DIR}/src/server/include + ${PDC_SOURCE_DIR}/src/server/pdc_server_region/include + ${PDC_SOURCE_DIR}/src/server/dablooms + ${PDC_SOURCE_DIR}/src/api/include + ${PDC_SOURCE_DIR}/src/api/pdc_obj/include + ${PDC_SOURCE_DIR}/src/api/pdc_region/include + ${PDC_SOURCE_DIR}/src/api/pdc_query/include + ${PDC_SOURCE_DIR}/src/api/pdc_transform/include + ${PDC_SOURCE_DIR}/src/api/pdc_analysis/include + ${PDC_SOURCE_DIR}/src/api/profiling/include + ${PDC_SOURCE_DIR}/src/utils/include + ${MERCURY_INCLUDE_DIR} + ${FASTBIT_INCLUDE_DIR} +) +message(STATUS "PDC_COMMON_INCLUDE_DIRS: ${PDC_COMMON_INCLUDE_DIRS}") +include_directories( + ${PROFILING_INCLUDE_DIRS} +) + +install( + FILES + ${CMAKE_BINARY_DIR}/pdc_config.h + DESTINATION + ${PDC_INSTALL_INCLUDE_DIR} + COMPONENT + headers +) + +#------------------------------------------------------------------------------ +# Options +#------------------------------------------------------------------------------ +#add_definitions(-DPDC_ENABLE_MPI=1) +#add_definitions(-DPDC_TIMING=1) +#add_definitions(-DPDC_ENABLE_CHECKPOINT=1) +#add_definitions(-DENABLE_MULTITHREAD=1) + +#------------------------------------------------------------------------------ +# Configure module header files +#------------------------------------------------------------------------------ +# Set unique vars used in the autogenerated config file (symbol import/export) +if(BUILD_SHARED_LIBS) + set(PDC_BUILD_SHARED_LIBS 1) + set(PDC_LIBTYPE SHARED) +else() + set(PDC_BUILD_SHARED_LIBS 0) + set(PDC_LIBTYPE STATIC) +endif() + +#------------------------------------------------------------------------------ +# Set sources +#------------------------------------------------------------------------------ +set(PDC_PROF_SRCS + ${CMAKE_CURRENT_SOURCE_DIR}/pdc_hashtab.c + ${CMAKE_CURRENT_SOURCE_DIR}/pdc_stack_ops.c + ) + +#------------------------------------------------------------------------------ +# Libraries +#------------------------------------------------------------------------------ +# PDCPROF +add_library(pdcprof ${PDC_PROF_SRCS}) +pdc_set_lib_options(pdcprof "pdcprof" ${PDC_LIBTYPE}) +target_include_directories(pdcprof PUBLIC "$" + $) + +set(PDC_EXPORTED_LIBS pdcprof ${PDC_EXPORTED_LIBS}) + +#----------------------------------------------------------------------------- +# Specify project header files to be installed +#----------------------------------------------------------------------------- +set(PDC_PROF_HEADERS + ${CMAKE_CURRENT_SOURCE_DIR}/include/pdc_hashtab.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/pdc_stack_ops.h + ) + +#----------------------------------------------------------------------------- +# Add file(s) to CMake Install +#----------------------------------------------------------------------------- +install( + FILES + ${PDC_PROF_HEADERS} + DESTINATION + ${PDC_INSTALL_INCLUDE_DIR} + COMPONENT + headers +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install +#----------------------------------------------------------------------------- +install( + TARGETS + pdcprof + EXPORT + ${PDC_EXPORTED_TARGETS} + LIBRARY DESTINATION ${PDC_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${PDC_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${PDC_INSTALL_BIN_DIR} +) + +#------------------------------------------------------------------------------ +# Set variables for parent scope +#------------------------------------------------------------------------------ +# Used by config.cmake.build.in and Testing +set(PDC_INCLUDES_BUILD_TIME + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + PARENT_SCOPE +) + +# Used by config.cmake.install.in +set(PDC_INCLUDES_INSTALL_TIME + ${PDC_INSTALL_INCLUDE_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + PARENT_SCOPE +) diff --git a/src/commons/profiling/include/pdc_hashtab.h b/src/commons/profiling/include/pdc_hashtab.h new file mode 100644 index 000000000..a664a8ce6 --- /dev/null +++ b/src/commons/profiling/include/pdc_hashtab.h @@ -0,0 +1,198 @@ +/* An expandable hash tables datatype. + Copyright (C) 1999-2017 Free Software Foundation, Inc. + Contributed by Vladimir Makarov (vmakarov@cygnus.com). + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA. */ + +/* This package implements basic hash table functionality. It is possible + to search for an entry, create an entry and destroy an entry. + + Elements in the table are generic pointers. + + The size of the table is not fixed; if the occupancy of the table + grows too high the hash table will be expanded. + + The abstract data implementation is based on generalized Algorithm D + from Knuth's book "The art of computer programming". Hash table is + expanded by creation of new hash table and transferring elements from + the old table to the new table. */ + +#ifndef __HASHTAB_H__ +#define __HASHTAB_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/* #include "ansidecl.h" */ +#define PTR void * + +/* The type for a hash code. */ +typedef unsigned int hashval_t; + +/* Callback function pointer types. */ + +/* Calculate hash of a table entry. */ +typedef hashval_t (*htab_hash)(const void *); + +/* Compare a table entry with a possible entry. The entry already in + the table always comes first, so the second element can be of a + different type (but in this case htab_find and htab_find_slot + cannot be used; instead the variants that accept a hash value + must be used). */ +typedef int (*htab_eq)(const void *, const void *); + +/* Cleanup function called whenever a live element is removed from + the hash table. */ +typedef void (*htab_del)(void *); + +/* Function called by htab_traverse for each live element. The first + arg is the slot of the element (which can be passed to htab_clear_slot + if desired), the second arg is the auxiliary pointer handed to + htab_traverse. Return 1 to continue scan, 0 to stop. */ +typedef int (*htab_trav)(void **, void *); + +/* Memory-allocation function, with the same functionality as calloc(). + Iff it returns NULL, the hash table implementation will pass an error + code back to the user, so if your code doesn't handle errors, + best if you use xcalloc instead. */ +typedef void *(*htab_alloc)(size_t, size_t); + +/* We also need a free() routine. */ +typedef void (*htab_free)(void *); + +/* Memory allocation and deallocation; variants which take an extra + argument. */ +typedef void *(*htab_alloc_with_arg)(void *, size_t, size_t); +typedef void (*htab_free_with_arg)(void *, void *); + +/* This macro defines reserved value for empty table entry. */ + +#define HTAB_EMPTY_ENTRY ((PTR)0) + +/* This macro defines reserved value for table entry which contained + a deleted element. */ + +#define HTAB_DELETED_ENTRY ((PTR)1) + +/* Hash tables are of the following type. The structure + (implementation) of this type is not needed for using the hash + tables. All work with hash table should be executed only through + functions mentioned below. The size of this structure is subject to + change. */ + +struct htab { + /* Pointer to hash function. */ + htab_hash hash_f; + + /* Pointer to comparison function. */ + htab_eq eq_f; + + /* Pointer to cleanup function. */ + htab_del del_f; + + /* Table itself. */ + void **entries; + + /* Current size (in entries) of the hash table. */ + size_t size; + + /* Current number of elements including also deleted elements. */ + size_t n_elements; + + /* Current number of deleted elements in the table. */ + size_t n_deleted; + + /* The following member is used for debugging. Its value is number + of all calls of `htab_find_slot' for the hash table. */ + unsigned int searches; + + /* The following member is used for debugging. Its value is number + of collisions fixed for time of work with the hash table. */ + unsigned int collisions; + + /* Pointers to allocate/free functions. */ + htab_alloc alloc_f; + htab_free free_f; + + /* Alternate allocate/free functions, which take an extra argument. */ + void * alloc_arg; + htab_alloc_with_arg alloc_with_arg_f; + htab_free_with_arg free_with_arg_f; + + /* Current size (in entries) of the hash table, as an index into the + table of primes. */ + unsigned int size_prime_index; +}; + +typedef struct htab *htab_t; + +/* An enum saying whether we insert into the hash table or not. */ +enum insert_option { NO_INSERT, INSERT }; + +/* The prototypes of the package functions. */ + +extern htab_t htab_create_alloc(size_t, htab_hash, htab_eq, htab_del, htab_alloc, htab_free); + +extern htab_t htab_create_alloc_ex(size_t, htab_hash, htab_eq, htab_del, void *, htab_alloc_with_arg, + htab_free_with_arg); + +extern htab_t htab_create_typed_alloc(size_t, htab_hash, htab_eq, htab_del, htab_alloc, htab_alloc, + htab_free); + +/* Backward-compatibility functions. */ +extern htab_t htab_create(size_t, htab_hash, htab_eq, htab_del); +extern htab_t htab_try_create(size_t, htab_hash, htab_eq, htab_del); + +extern void htab_set_functions_ex(htab_t, htab_hash, htab_eq, htab_del, void *, htab_alloc_with_arg, + htab_free_with_arg); + +extern void htab_delete(htab_t); +extern void htab_empty(htab_t); + +extern void * htab_find(htab_t, const void *); +extern void **htab_find_slot(htab_t, const void *, enum insert_option); +extern void * htab_find_with_hash(htab_t, const void *, hashval_t); +extern void **htab_find_slot_with_hash(htab_t, const void *, hashval_t, enum insert_option); +extern void htab_clear_slot(htab_t, void **); +extern void htab_remove_elt(htab_t, void *); +extern void htab_remove_elt_with_hash(htab_t, void *, hashval_t); + +extern void htab_traverse(htab_t, htab_trav, void *); +extern void htab_traverse_noresize(htab_t, htab_trav, void *); + +extern size_t htab_size(htab_t); +extern size_t htab_elements(htab_t); +extern double htab_collisions(htab_t); + +/* A hash function for pointers. */ +extern htab_hash htab_hash_pointer; + +/* An equality function for pointers. */ +extern htab_eq htab_eq_pointer; + +/* A hash function for null-terminated strings. */ +extern hashval_t htab_hash_string(const void *); + +/* An iterative hash function for arbitrary data. */ +extern hashval_t iterative_hash(const void *, size_t, hashval_t); +/* Shorthand for hashing something with an intrinsic size. */ +#define iterative_hash_object(OB, INIT) iterative_hash(&OB, sizeof(OB), INIT) + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __HASHTAB_H */ diff --git a/src/commons/profiling/include/pdc_stack_ops.h b/src/commons/profiling/include/pdc_stack_ops.h new file mode 100644 index 000000000..110d03900 --- /dev/null +++ b/src/commons/profiling/include/pdc_stack_ops.h @@ -0,0 +1,70 @@ +#ifndef _STACK_OPS_H +#define _STACK_OPS_H + +#include "pdc_config.h" +#include "pdc_private.h" +#include +#include +#include +#include +#include + +typedef void *hash_table_t; + +typedef struct profileEntry { + struct profileEntry *next; + struct profileEntry *prev; + const char * ftnkey; + const char * tags; + int64_t count; + int64_t localTotal; + int64_t CumTotal; + int64_t locmin; + int64_t locmax; + double usecTotal; + struct timespec callTime; + struct timespec startTime; + struct timespec totalTime; + struct timespec selfTime; + + struct profileEntry *parent; +} profileEntry_t; + +// typedef enum _boolean {FALSE = 0, TRUE} bool_t; +extern pbool_t enableProfiling; + +#ifndef RESET_TIMER +#define RESET_TIMER(x) (x).tv_sec = (x).tv_nsec = 0; +#endif + +#ifndef TIMER_DIFF +/* t0 = t1 - t2 */ +#define TIMER_DIFF(t0, t1, t2) \ + { \ + if (t2.tv_nsec > (t1).tv_nsec) { \ + (t1).tv_nsec += 1000000000; \ + (t1).tv_sec -= 1; \ + } \ + (t0).tv_sec = (t1).tv_sec - (t2).tv_sec; \ + (t0).tv_nsec = (t1).tv_nsec - (t2).tv_nsec; \ + } +#endif + +#ifndef TIMER_ADD +/* t0 += t1 */ +#define TIMER_ADD(t0, t1) \ + { \ + (t0).tv_sec += (t1).tv_sec; \ + if (((t0).tv_nsec += (t1).tv_nsec) > 10000000000) { \ + (t0).tv_sec += 1; \ + (t0).tv_nsec -= 10000000000; \ + } \ + } +#endif + +void initialize_profile(void **table, size_t tabsize); +void finalize_profile(); +void push(const char *ftnkey, const char *tags); +void pop(); + +#endif diff --git a/src/commons/profiling/pdc_hashtab.c b/src/commons/profiling/pdc_hashtab.c new file mode 100644 index 000000000..e59b7e3ba --- /dev/null +++ b/src/commons/profiling/pdc_hashtab.c @@ -0,0 +1,540 @@ +/* An expandable hash tables datatype. + Copyright (C) 1999, 2000, 2001, 2002 Free Software Foundation, Inc. + Contributed by Vladimir Makarov (vmakarov@cygnus.com). + +This file is part of the libiberty library. +Libiberty is free software; you can redistribute it and/or +modify it under the terms of the GNU Library General Public +License as published by the Free Software Foundation; either +version 2 of the License, or (at your option) any later version. + +Libiberty is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Library General Public License for more details. + +You should have received a copy of the GNU Library General Public +License along with libiberty; see the file COPYING.LIB. If +not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, +Boston, MA 02111-1307, USA. */ + +/* This package implements basic hash table functionality. It is possible + to search for an entry, create an entry and destroy an entry. + + Elements in the table are generic pointers. + + The size of the table is not fixed; if the occupancy of the table + grows too high the hash table will be expanded. + + The abstract data implementation is based on generalized Algorithm D + from Knuth's book "The art of computer programming". Hash table is + expanded by creation of new hash table and transferring elements from + the old table to the new table. */ + +#include +#include +#include +#include +#include "pdc_config.h" +#include "pdc_hashtab.h" + +/* This macro defines reserved value for empty table entry. */ + +#define EMPTY_ENTRY ((PTR)0) + +/* This macro defines reserved value for table entry which contained + a deleted element. */ + +#define DELETED_ENTRY ((PTR)1) + +static unsigned long higher_prime_number(unsigned long); +static hashval_t hash_pointer(const void *); +static int eq_pointer(const void *, const void *); +static int htab_expand(htab_t); +static PTR * find_empty_slot_for_expand(htab_t, hashval_t); + +/* At some point, we could make these be NULL, and modify the + hash-table routines to handle NULL specially; that would avoid + function-call overhead for the common case of hashing pointers. */ +htab_hash htab_hash_pointer = hash_pointer; +htab_eq htab_eq_pointer = eq_pointer; + +/* The following function returns a nearest prime number which is + greater than N, and near a power of two. */ + +static unsigned long higher_prime_number(n) unsigned long n; +{ + /* These are primes that are near, but slightly smaller than, a + power of two. */ + static const unsigned long primes[] = { + (unsigned long)7, + (unsigned long)13, + (unsigned long)31, + (unsigned long)61, + (unsigned long)127, + (unsigned long)251, + (unsigned long)509, + (unsigned long)1021, + (unsigned long)2039, + (unsigned long)4093, + (unsigned long)8191, + (unsigned long)16381, + (unsigned long)32749, + (unsigned long)65521, + (unsigned long)131071, + (unsigned long)262139, + (unsigned long)524287, + (unsigned long)1048573, + (unsigned long)2097143, + (unsigned long)4194301, + (unsigned long)8388593, + (unsigned long)16777213, + (unsigned long)33554393, + (unsigned long)67108859, + (unsigned long)134217689, + (unsigned long)268435399, + (unsigned long)536870909, + (unsigned long)1073741789, + (unsigned long)2147483647, + /* 4294967291L */ + ((unsigned long)2147483647) + ((unsigned long)2147483644), + }; + + const unsigned long *low = &primes[0]; + const unsigned long *high = &primes[sizeof(primes) / sizeof(primes[0])]; + + while (low != high) { + const unsigned long *mid = low + (high - low) / 2; + if (n > *mid) + low = mid + 1; + else + high = mid; + } + + /* If we've run out of primes, abort. */ + if (n > *low) { + fprintf(stderr, "Cannot find prime bigger than %lu\n", n); + abort(); + } + + return *low; +} + +/* Returns a hash code for P. */ + +static hashval_t hash_pointer(p) const PTR p; +{ + return (hashval_t)((long)p >> 3); +} + +/* Returns non-zero if P1 and P2 are equal. */ + +static int eq_pointer(p1, p2) const PTR p1; +const PTR p2; +{ + return p1 == p2; +} + +/* This function creates table with length slightly longer than given + source length. Created hash table is initiated as empty (all the + hash table entries are EMPTY_ENTRY). The function returns the + created hash table, or NULL if memory allocation fails. */ + +htab_t htab_create_alloc(size, hash_f, eq_f, del_f, alloc_f, free_f) size_t size; +htab_hash hash_f; +htab_eq eq_f; +htab_del del_f; +htab_alloc alloc_f; +htab_free free_f; +{ + htab_t result; + size = higher_prime_number(size); + result = (htab_t)(*alloc_f)(1, sizeof(struct htab)); + if (result == NULL) + return NULL; + + result->entries = (PTR *)(*alloc_f)(size, sizeof(PTR)); + if (result->entries == NULL) { + if (free_f != NULL) + (*free_f)(result); + return NULL; + } + result->size = size; + result->hash_f = hash_f; + result->eq_f = eq_f; + result->del_f = del_f; + result->alloc_f = alloc_f; + result->free_f = free_f; + + return result; +} + +/* These functions exist solely for backward compatibility. */ + +#undef htab_create +htab_t htab_create(size, hash_f, eq_f, del_f) size_t size; +htab_hash hash_f; +htab_eq eq_f; +htab_del del_f; +{ + return htab_create_alloc(size, hash_f, eq_f, del_f, calloc, free); +} + +htab_t htab_try_create(size, hash_f, eq_f, del_f) size_t size; +htab_hash hash_f; +htab_eq eq_f; +htab_del del_f; +{ + return htab_create_alloc(size, hash_f, eq_f, del_f, calloc, free); +} + +/* This function frees all memory allocated for given hash table. + Naturally the hash table must already exist. */ + +void htab_delete(htab) htab_t htab; +{ + int i; + + if (htab->del_f) { + for (i = htab->size - 1; i >= 0; i--) + if (htab->entries[i] != EMPTY_ENTRY && htab->entries[i] != DELETED_ENTRY) + (*htab->del_f)(htab->entries[i]); + } + if (htab->free_f != NULL) { + (*htab->free_f)(htab->entries); + (*htab->free_f)(htab); + } +} + +/* This function clears all entries in the given hash table. */ + +void htab_empty(htab) htab_t htab; +{ + int i; + + if (htab->del_f) + for (i = htab->size - 1; i >= 0; i--) + if (htab->entries[i] != EMPTY_ENTRY && htab->entries[i] != DELETED_ENTRY) + (*htab->del_f)(htab->entries[i]); + + memset(htab->entries, 0, htab->size * sizeof(PTR)); +} + +/* Similar to htab_find_slot, but without several unwanted side effects: + - Does not call htab->eq_f when it finds an existing entry. + - Does not change the count of elements/searches/collisions in the + hash table. + This function also assumes there are no deleted entries in the table. + HASH is the hash value for the element to be inserted. */ + +static PTR *find_empty_slot_for_expand(htab, hash) htab_t htab; +hashval_t hash; +{ + size_t size = htab->size; + unsigned int index = hash % size; + PTR * slot = htab->entries + index; + hashval_t hash2; + + if (*slot == EMPTY_ENTRY) + return slot; + else if (*slot == DELETED_ENTRY) + abort(); + + hash2 = 1 + hash % (size - 2); + for (;;) { + index += hash2; + if (index >= size) + index -= size; + + slot = htab->entries + index; + if (*slot == EMPTY_ENTRY) + return slot; + else if (*slot == DELETED_ENTRY) + abort(); + } +} + +/* The following function changes size of memory allocated for the + entries and repeatedly inserts the table elements. The occupancy + of the table after the call will be about 50%. Naturally the hash + table must already exist. Remember also that the place of the + table entries is changed. If memory allocation failures are allowed, + this function will return zero, indicating that the table could not be + expanded. If all goes well, it will return a non-zero value. */ + +static int htab_expand(htab) htab_t htab; +{ + PTR *oentries; + PTR *olimit; + PTR *p; + PTR *nentries; + + oentries = htab->entries; + olimit = oentries + htab->size; + + htab->size = higher_prime_number(htab->size * 2); + + nentries = (PTR *)(*htab->alloc_f)(htab->size, sizeof(PTR *)); + if (nentries == NULL) + return 0; + htab->entries = nentries; + htab->n_elements -= htab->n_deleted; + htab->n_deleted = 0; + + p = oentries; + do { + PTR x = *p; + + if (x != EMPTY_ENTRY && x != DELETED_ENTRY) { + PTR *q = find_empty_slot_for_expand(htab, (*htab->hash_f)(x)); + *q = x; + } + p++; + } while (p < olimit); + + if (htab->free_f != NULL) + (*htab->free_f)(oentries); + + return 1; +} + +/* This function searches for a hash table entry equal to the given + element. It cannot be used to insert or delete an element. */ + +PTR htab_find_with_hash(htab, element, hash) htab_t htab; +const PTR element; +hashval_t hash; +{ + unsigned int index; + hashval_t hash2; + size_t size; + PTR entry; + + htab->searches++; + size = htab->size; + index = hash % size; + + entry = htab->entries[index]; + if (entry == EMPTY_ENTRY || (entry != DELETED_ENTRY && (*htab->eq_f)(entry, element))) + return entry; + + hash2 = 1 + hash % (size - 2); + + for (;;) { + htab->collisions++; + index += hash2; + if (index >= size) + index -= size; + + entry = htab->entries[index]; + if (entry == EMPTY_ENTRY || (entry != DELETED_ENTRY && (*htab->eq_f)(entry, element))) + return entry; + } +} + +/* Like htab_find_slot_with_hash, but compute the hash value from the + element. */ + +PTR htab_find(htab, element) htab_t htab; +const PTR element; +{ + return htab_find_with_hash(htab, element, (*htab->hash_f)(element)); +} + +/* This function searches for a hash table slot containing an entry + equal to the given element. To delete an entry, call this with + INSERT = 0, then call htab_clear_slot on the slot returned (possibly + after doing some checks). To insert an entry, call this with + INSERT = 1, then write the value you want into the returned slot. + When inserting an entry, NULL may be returned if memory allocation + fails. */ + +PTR * htab_find_slot_with_hash(htab, element, hash, insert) htab_t htab; +const PTR element; +hashval_t hash; +enum insert_option insert; +{ + PTR * first_deleted_slot; + unsigned int index; + hashval_t hash2; + size_t size; + PTR entry; + + if (insert == INSERT && htab->size * 3 <= htab->n_elements * 4 && htab_expand(htab) == 0) + return NULL; + + size = htab->size; + index = hash % size; + + htab->searches++; + first_deleted_slot = NULL; + + entry = htab->entries[index]; + if (entry == EMPTY_ENTRY) + goto empty_entry; + else if (entry == DELETED_ENTRY) + first_deleted_slot = &htab->entries[index]; + else if ((*htab->eq_f)(entry, element)) + return &htab->entries[index]; + + hash2 = 1 + hash % (size - 2); + for (;;) { + htab->collisions++; + index += hash2; + if (index >= size) + index -= size; + + entry = htab->entries[index]; + if (entry == EMPTY_ENTRY) + goto empty_entry; + else if (entry == DELETED_ENTRY) { + if (!first_deleted_slot) + first_deleted_slot = &htab->entries[index]; + } + else if ((*htab->eq_f)(entry, element)) + return &htab->entries[index]; + } + +empty_entry: + if (insert == NO_INSERT) + return NULL; + + htab->n_elements++; + + if (first_deleted_slot) { + *first_deleted_slot = EMPTY_ENTRY; + return first_deleted_slot; + } + + return &htab->entries[index]; +} + +/* Like htab_find_slot_with_hash, but compute the hash value from the + element. */ + +PTR * htab_find_slot(htab, element, insert) htab_t htab; +const PTR element; +enum insert_option insert; +{ + return htab_find_slot_with_hash(htab, element, (*htab->hash_f)(element), insert); +} + +/* This function deletes an element with the given value from hash + table. If there is no matching element in the hash table, this + function does nothing. */ + +void htab_remove_elt(htab, element) htab_t htab; +PTR element; +{ + PTR *slot; + + slot = htab_find_slot(htab, element, NO_INSERT); + if (*slot == EMPTY_ENTRY) + return; + + if (htab->del_f) + (*htab->del_f)(*slot); + + *slot = DELETED_ENTRY; + htab->n_deleted++; +} + +/* This function clears a specified slot in a hash table. It is + useful when you've already done the lookup and don't want to do it + again. */ + +void htab_clear_slot(htab, slot) htab_t htab; +PTR *slot; +{ + if (slot < htab->entries || slot >= htab->entries + htab->size || *slot == EMPTY_ENTRY || + *slot == DELETED_ENTRY) + abort(); + + if (htab->del_f) + (*htab->del_f)(*slot); + + *slot = DELETED_ENTRY; + htab->n_deleted++; +} + +/* This function scans over the entire hash table calling + CALLBACK for each live entry. If CALLBACK returns false, + the iteration stops. INFO is passed as CALLBACK's second + argument. */ + +void htab_traverse(htab, callback, info) htab_t htab; +htab_trav callback; +PTR info; +{ + PTR *slot = htab->entries; + PTR *limit = slot + htab->size; + + do { + PTR x = *slot; + if (x != EMPTY_ENTRY && x != DELETED_ENTRY) + if (!(*callback)(slot, info)) + break; + } while (++slot < limit); +} + +/* Return the current size of given hash table. */ + +size_t htab_size(htab) htab_t htab; +{ + return htab->size; +} + +/* Return the current number of elements in given hash table. */ + +size_t htab_elements(htab) htab_t htab; +{ + return htab->n_elements - htab->n_deleted; +} + +/* Return the fraction of fixed collisions during all work with given + hash table. */ + +double htab_collisions(htab) htab_t htab; +{ + if (htab->searches == 0) + return 0.0; + + return (double)htab->collisions / (double)htab->searches; +} + +/* Hash P as a null-terminated string. + + Copied from gcc/hashtable.c. Zack had the following to say with respect + to applicability, though note that unlike hashtable.c, this hash table + implementation re-hashes rather than chain buckets. + + http://gcc.gnu.org/ml/gcc-patches/2001-08/msg01021.html + From: Zack Weinberg + Date: Fri, 17 Aug 2001 02:15:56 -0400 + + I got it by extracting all the identifiers from all the source code + I had lying around in mid-1999, and testing many recurrences of + the form "H_n = H_{n-1} * K + c_n * L + M" where K, L, M were either + prime numbers or the appropriate identity. This was the best one. + I don't remember exactly what constituted "best", except I was + looking at bucket-length distributions mostly. + + So it should be very good at hashing identifiers, but might not be + as good at arbitrary strings. + + I'll add that it thoroughly trounces the hash functions recommended + for this use at http://burtleburtle.net/bob/hash/index.html, both + on speed and bucket distribution. I haven't tried it against the + function they just started using for Perl's hashes. */ + +hashval_t htab_hash_string(p) const PTR p; +{ + const unsigned char *str = (const unsigned char *)p; + hashval_t r = 0; + unsigned char c; + + while ((c = *str++) != 0) + r = r * 67 + c - 113; + + return r; +} diff --git a/src/commons/profiling/pdc_stack_ops.c b/src/commons/profiling/pdc_stack_ops.c new file mode 100644 index 000000000..fab9f274a --- /dev/null +++ b/src/commons/profiling/pdc_stack_ops.c @@ -0,0 +1,264 @@ +#include +#include +#include +#include "pdc_stack_ops.h" +#include "pdc_hashtab.h" + +profileEntry_t *calltree = NULL; +profileEntry_t *freelist = NULL; + +static int profilerrors = 0; + +hash_table_t hashtable; + +htab_t thisHashTable; + +/* For now we disable profiling (by default) + * Note that one can always ENABLE it by set the + * environment variable "PROFILE_ENABLE=true" + */ +pbool_t enableProfiling = FALSE; + +/* + * The idea of this implementation is to simulate the call stack + * of the running application. Each function that we care about + * begins with a FUNC_ENTER(x) declaration and finishes with + * FUNC_LEAVE(ret). These of course are macros and under + * the condition that we enable profiling, these expand into + * push and pop operations which we define below. + * + * Example: suppose that a user application is defined as follows + * int main() { + * a(); + * b(); + * c(); + * return 0; + * } + * + * void a() { + * aa(); + * aaa(); + * ab(); + * } + * + * void b() { + * bb(); + * bbb(); + * bc(); + * } + * + * void c() { + * cc(); + * ccc(); + * ca(); + * } + * + * Assume that all of the internal functions only make system calls + * or 3rd party libraries, i.e. the underlying functions will NOT + * be profiled. + * + * The implementation of stack_ops will maintain a call tree + * that mirrors that of the actual program, i.e. the alltree data + * structure will contain something like the following as we enter + * the first function contained by a(): + * + * ("main") --> ("a") --> ("aa") + * + * The entry for "main" has a /start_time and no /total_time + * Similarly, "a" has it's own /start_time and no /total_time + * The final entry: "aa" has a start-time and just prior to + * the return to it's parent ("a"), we sample the real-time + * clock as part of the POP functionality. Using the current + * time minus the start-time we establish the raw total elapsed + * time for the current function. + * NOTE: The actual runtime spent within the function is + * a calculation which subtracts out the total elapsed times + * of all of the lower-level functions, e.g. suppose ("a") + * has a total runtime of 10. If the total runtime of ("aa") + * in the simple call chain shown above is 5, then the actual + * profiled time spent in ("a") is 10 - 5 = 5. + * Ultimately, if were to execute the entire program and then + * sum all of the individual profile times, the total should + * match the execution time of the program. + */ + +void +push(const char *ftnkey, const char *tags) +{ + profileEntry_t *thisEntry; + if (freelist != NULL) { + thisEntry = freelist; + freelist = thisEntry->next; + } + else { + if ((thisEntry = (profileEntry_t *)malloc(sizeof(profileEntry_t))) == NULL) { + perror("malloc"); + profilerrors++; + } + } + + if (profilerrors) + return; + thisEntry->ftnkey = ftnkey; + thisEntry->tags = tags; + thisEntry->prev = calltree; + thisEntry->next = NULL; + calltree = thisEntry; + + /* Timing */ + clock_gettime(CLOCK_REALTIME, &thisEntry->startTime); + RESET_TIMER(thisEntry->callTime); + return; +} + +void +pop() +{ + struct timespec current_time; + profileEntry_t *master; + profileEntry_t *thisEntry = calltree; + int update_entry = TRUE; + if (thisEntry == NULL) + return; /* This shouldn't happen */ + + /* Timing */ + clock_gettime(CLOCK_REALTIME, ¤t_time); + TIMER_DIFF(thisEntry->totalTime, current_time, thisEntry->startTime); + TIMER_DIFF(thisEntry->selfTime, thisEntry->totalTime, thisEntry->callTime); + calltree = thisEntry->prev; + if (calltree != NULL) { + TIMER_ADD(calltree->callTime, thisEntry->totalTime); + } + /* Check to see if this function has already been added to the hashtable */ + void **tableEntry = htab_find_slot(thisHashTable, thisEntry, INSERT); + if (*tableEntry == NULL) { + /* No table entry found so add it now ... */ + master = (profileEntry_t *)malloc(sizeof(profileEntry_t)); + if (master) { + thisEntry->count = 1; + memcpy(master, thisEntry, sizeof(profileEntry_t)); + *tableEntry = master; + } + update_entry = FALSE; + } + + if (update_entry) { + master = *(profileEntry_t **)tableEntry; + master->count++; + TIMER_ADD(master->totalTime, thisEntry->totalTime); + TIMER_ADD(master->selfTime, thisEntry->selfTime); + } + + /* Rather than freeing the container, we add the + * current entry onto the freelist. + */ + thisEntry->next = freelist; + freelist = thisEntry; +} + +hashval_t +hash_profile_entry(const void *p) +{ + const profileEntry_t *thisEntry = (const profileEntry_t *)p; + return htab_hash_string(thisEntry->ftnkey); +} + +int +eq_profile_entry(const void *a, const void *b) +{ + const profileEntry_t *tp_a = (const profileEntry_t *)a; + const profileEntry_t *tp_b = (const profileEntry_t *)b; + return (tp_a->ftnkey == tp_b->ftnkey); +} + +void +initialize_profile(void **hashtab, size_t size) +{ + if (*hashtab == NULL) { + if ((thisHashTable = htab_try_create(size, hash_profile_entry, eq_profile_entry, free)) == NULL) { + return; + } + *hashtab = thisHashTable; + } +} + +int +show_profile_info(void **ht_live_entry, void *extraInfo ATTRIBUTE(unused)) +{ + static int count = 0; + char * LineBreak = "------------------------------------------------------------------------------"; + char * header = " item calls Time/call [Sec,nSec]\tftn_name"; + const profileEntry_t *thisEntry = *(const profileEntry_t **)ht_live_entry; + + if (thisEntry) { + struct timespec totalTime; + int64_t totalCalls = thisEntry->count; + if (count == 0) + puts(header); + totalTime = thisEntry->totalTime; + printf("%s\n %d\t%-6" PRId64 " %6" PRId64 ",%6" PRId64 "\t\t %s\n", LineBreak, ++count, totalCalls, + totalTime.tv_sec / totalCalls, totalTime.tv_nsec / totalCalls, thisEntry->ftnkey); + } + + return TRUE; +} + +/* Returns 1 if we set enableProfiling to TRUE + * otherwise returns 0. + */ +int +toggle_profile_enable() +{ + if (enableProfiling == FALSE) + enableProfiling = TRUE; + else + enableProfiling = FALSE; + + return (enableProfiling ? 1 : 0); +} + +/* These functions should be used when we've actually built the profiler as a shared library. + * Note: One might check an environment variable to see if a non-default size + * for the hashtable initialization should be used... + * The profile_fini should probably be used to dump the contents of the profile + * hashtable. + */ + +void __attribute__((constructor)) profile_init(void) +{ + int default_HashtableSize = 128; + char *size_override = NULL; + char *profile_enable = getenv("PROFILE_ENABLE"); + if (profile_enable != NULL) { + if (strcasecmp(profile_enable, "true") == 0) { + enableProfiling = TRUE; + } + else if (strcasecmp(profile_enable, "false") == 0) { + enableProfiling = FALSE; + } + } + // While it is tempting to skip creating a hashtable + // if we've disabled profiling (see above), I want + // to give the user the ability at runtime to + // possibly enable everything... + // I don't currently include any APIs to enable + // or disable profiling at runtime, but that is + // on the TODO list. + + size_override = getenv("PROFILE_HASHTABLESIZE"); + if (size_override != NULL) { + int override_value = atoi(size_override); + if (override_value > 0) { + default_HashtableSize = override_value; + } + } + initialize_profile(&hashtable, default_HashtableSize); +} + +void __attribute__((destructor)) finalize_profile(void) +{ + int count = 1; + if (thisHashTable != NULL) { + htab_traverse(thisHashTable, show_profile_info, &count); + } +} diff --git a/src/commons/serde/include/pdc_serde.h b/src/commons/serde/include/pdc_serde.h index 6211e1917..8c5efae9e 100644 --- a/src/commons/serde/include/pdc_serde.h +++ b/src/commons/serde/include/pdc_serde.h @@ -62,11 +62,11 @@ void pdc_serde_append_key_value(PDC_SERDE_SerializedData *data, PDC_SERDE_Key *k /** * @brief get the total size of PDC_SERDE_SerializedData structure instance - * + * * @param data Pointer to the PDC_SERDE_SerializedData structure instance - * + * * @return total size of the PDC_SERDE_SerializedData structure instance -*/ + */ size_t get_total_size_for_serialized_data(PDC_SERDE_SerializedData *data); /** @@ -113,9 +113,9 @@ void pdc_serde_print(PDC_SERDE_SerializedData *data); static inline PDC_SERDE_Key * PDC_SERDE_KEY(void *key, PDC_CType pdc_type, size_t size) { - PDC_SERDE_Key *pdc_key = (PDC_SERDE_Key *)malloc(sizeof(PDC_SERDE_Key)); - size_t key_size = (size_t) get_size_by_class_n_type(key, size, PDC_CLS_SCALAR, pdc_type); - pdc_key->key = malloc(key_size); + PDC_SERDE_Key *pdc_key = (PDC_SERDE_Key *)malloc(sizeof(PDC_SERDE_Key)); + size_t key_size = (size_t)get_size_by_class_n_type(key, size, PDC_CLS_SCALAR, pdc_type); + pdc_key->key = malloc(key_size); memcpy(pdc_key->key, key, key_size); pdc_key->pdc_type = pdc_type; pdc_key->size = key_size; @@ -135,14 +135,15 @@ PDC_SERDE_KEY(void *key, PDC_CType pdc_type, size_t size) static inline PDC_SERDE_Value * PDC_SERDE_VALUE(void *data, PDC_CType pdc_type, PDC_CType_Class pdc_class, size_t size) { - PDC_SERDE_Value *pdc_value = (PDC_SERDE_Value *)malloc(sizeof(PDC_SERDE_Value)); - size_t value_size = 0; + PDC_SERDE_Value *pdc_value = (PDC_SERDE_Value *)malloc(sizeof(PDC_SERDE_Value)); + size_t value_size = 0; if (pdc_class == PDC_CLS_STRUCT) { // TODO: we need to check if data is a valid PDC_SERDE_SerializedData structure. PDC_SERDE_SerializedData *struct_data = (PDC_SERDE_SerializedData *)data; size = struct_data->totalSize; - } else { - value_size = (size_t) get_size_by_class_n_type(data, size, pdc_class, pdc_type); + } + else { + value_size = (size_t)get_size_by_class_n_type(data, size, pdc_class, pdc_type); } pdc_value->data = malloc(value_size); memcpy(pdc_value->data, data, value_size); diff --git a/src/commons/serde/pdc_serde.c b/src/commons/serde/pdc_serde.c index e0959b5cd..fe9e9aa64 100644 --- a/src/commons/serde/pdc_serde.c +++ b/src/commons/serde/pdc_serde.c @@ -28,10 +28,12 @@ pdc_serde_append_key_value(PDC_SERDE_SerializedData *data, PDC_SERDE_Key *key, P data->data->totalSize += (sizeof(int) + sizeof(int) + sizeof(size_t) + value->size); } -size_t get_total_size_for_serialized_data(PDC_SERDE_SerializedData *data) { +size_t +get_total_size_for_serialized_data(PDC_SERDE_SerializedData *data) +{ if (data->totalSize <= 0) { size_t total_size = data->header->totalSize + data->data->totalSize + sizeof(size_t) * 6; - data->totalSize = total_size; + data->totalSize = total_size; } return data->totalSize; } @@ -287,13 +289,12 @@ test_serde_framework() char * doubleKey_str = "double"; double doubleVal = 3.14159; PDC_SERDE_Key * doubleKey = PDC_SERDE_KEY(doubleKey_str, PDC_STRING, sizeof(doubleKey_str)); - PDC_SERDE_Value *doubleValue = - PDC_SERDE_VALUE(&doubleVal, PDC_DOUBLE, PDC_CLS_SCALAR, sizeof(double)); + PDC_SERDE_Value *doubleValue = PDC_SERDE_VALUE(&doubleVal, PDC_DOUBLE, PDC_CLS_SCALAR, sizeof(double)); pdc_serde_append_key_value(data, doubleKey, doubleValue); - char * strKey_str = "string"; - char * strVal = "Hello, World!"; - PDC_SERDE_Key *strKey = PDC_SERDE_KEY(strKey_str, PDC_STRING, (strlen(strKey_str) + 1) * sizeof(char)); + char * strKey_str = "string"; + char * strVal = "Hello, World!"; + PDC_SERDE_Key * strKey = PDC_SERDE_KEY(strKey_str, PDC_STRING, (strlen(strKey_str) + 1) * sizeof(char)); PDC_SERDE_Value *strValue = PDC_SERDE_VALUE(strVal, PDC_STRING, PDC_CLS_SCALAR, (strlen(strVal) + 1) * sizeof(char)); pdc_serde_append_key_value(data, strKey, strValue); @@ -314,7 +315,7 @@ test_serde_framework() PDC_SERDE_SerializedData *point_data = pdc_serde_init(2); PDC_SERDE_Key * x_name = PDC_SERDE_KEY("x", PDC_STRING, sizeof(char *)); - PDC_SERDE_Value *x_value = PDC_SERDE_VALUE(&pointVal.x, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); + PDC_SERDE_Value * x_value = PDC_SERDE_VALUE(&pointVal.x, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); PDC_SERDE_Key * y_name = PDC_SERDE_KEY("y", PDC_STRING, sizeof(char *)); PDC_SERDE_Value *y_value = PDC_SERDE_VALUE(&pointVal.y, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); @@ -323,9 +324,8 @@ test_serde_framework() pdc_serde_append_key_value(point_data, y_name, y_value); void *point_buffer = pdc_serde_serialize(point_data); - PDC_SERDE_Key * structKey = PDC_SERDE_KEY(pointKey, PDC_STRING, sizeof(pointKey)); - PDC_SERDE_Value *structValue = - PDC_SERDE_VALUE(point_buffer, PDC_VOID_PTR, PDC_CLS_STRUCT, sizeof(Point)); + PDC_SERDE_Key * structKey = PDC_SERDE_KEY(pointKey, PDC_STRING, sizeof(pointKey)); + PDC_SERDE_Value *structValue = PDC_SERDE_VALUE(point_buffer, PDC_VOID_PTR, PDC_CLS_STRUCT, sizeof(Point)); pdc_serde_append_key_value(data, structKey, structValue); // Serialize the data diff --git a/src/commons/utils/include/pdc_id_pkg.h b/src/commons/utils/include/pdc_id_pkg.h new file mode 100644 index 000000000..9623661fd --- /dev/null +++ b/src/commons/utils/include/pdc_id_pkg.h @@ -0,0 +1,57 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#ifndef PDC_ID_PKG_H +#define PDC_ID_PKG_H + +#include "pdc_private.h" +#include "pdc_linkedlist.h" +#include "mercury_atomic.h" +/* + * Number of bits to use for ID Type in each atom. Increase if more types + * are needed (though this will decrease the number of available IDs per + * type). This is the only number that must be changed since all other bit + * field sizes and masks are calculated from TYPE_BITS. + */ +#define TYPE_BITS 8 +#define TYPE_MASK (((pdcid_t)1 << TYPE_BITS) - 1) +#define PDC_MAX_NUM_TYPES TYPE_MASK +/* + * Number of bits to use for the Atom index in each atom (assumes 8-bit + * bytes). We don't use the sign bit. + */ +#define ID_BITS ((sizeof(pdcid_t) * 8) - (TYPE_BITS + 1)) +#define ID_MASK (((pdcid_t)1 << ID_BITS) - 1) + +/* Map an atom to an ID type number */ +#define PDC_TYPE(a) ((PDC_type_t)(((pdcid_t)(a) >> ID_BITS) & TYPE_MASK)) + +struct _pdc_id_info { + pdcid_t id; /* ID for this info */ + hg_atomic_int32_t count; /* ref. count for this atom */ + void * obj_ptr; /* pointer associated with the atom */ + PDC_LIST_ENTRY(_pdc_id_info) entry; +}; + +#endif /* PDC_ID_PKG_H */ diff --git a/src/commons/utils/include/pdc_linkedlist.h b/src/commons/utils/include/pdc_linkedlist.h new file mode 100644 index 000000000..a9de691b0 --- /dev/null +++ b/src/commons/utils/include/pdc_linkedlist.h @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2013-2016 Argonne National Laboratory, Department of Energy, + * UChicago Argonne, LLC and The HDF Group. + * All rights reserved. + * + * The full copyright notice, including terms governing use, modification, + * and redistribution, is contained in the COPYING file that can be + * found at the root of the source code distribution tree. + */ + +/* Code below is derived from sys/queue.h which follows the below notice: + * + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)queue.h 8.5 (Berkeley) 8/20/94 + */ + +#ifndef PDC_LINKEDLIST_H +#define PDC_LINKEDLIST_H + +// #include "pdc_cont_pkg.h" +// #include "pdc_cont.h" +#include "mercury_thread_mutex.h" +#include + +#define PDC_LIST_HEAD_INITIALIZER(name) \ + { \ + NULL \ + } + +#define PDC_LIST_HEAD_INIT(struct_head_name, var_name) \ + struct struct_head_name var_name = PDC_LIST_HEAD_INITIALIZER(var_name) + +#define PDC_LIST_HEAD_DECL(struct_head_name, struct_entry_name) \ + struct struct_head_name { \ + struct struct_entry_name *head; \ + } + +#define PDC_LIST_HEAD(struct_entry_name) \ + struct { \ + struct struct_entry_name *head; \ + hg_thread_mutex_t lock; \ + } + +#define PDC_LIST_ENTRY(struct_entry_name) \ + struct { \ + struct struct_entry_name * next; \ + struct struct_entry_name **prev; \ + } + +#define PDC_LIST_INIT(head_ptr) \ + do { \ + (head_ptr)->head = NULL; \ + hg_thread_mutex_init(&(head_ptr)->lock); \ + } while (/*CONSTCOND*/ 0) + +#define PDC_LIST_IS_EMPTY(head_ptr) ((head_ptr)->head == NULL) + +#define PDC_LIST_FIRST(head_ptr) ((head_ptr)->head) + +#define PDC_LIST_GET_FIRST(var, head_ptr) (var = (head_ptr)->head) + +#define PDC_LIST_NEXT(entry_ptr, entry_field_name) ((entry_ptr)->entry_field_name.next) + +#define PDC_LIST_TO_NEXT(entry_ptr, entry_field_name) ((entry_ptr) = (entry_ptr)->entry_field_name.next) + +#define PDC_LIST_INSERT_HEAD(head_ptr, entry_ptr, entry_field_name) \ + do { \ + if (((entry_ptr)->entry_field_name.next = (head_ptr)->head) != NULL) \ + (head_ptr)->head->entry_field_name.prev = &(entry_ptr)->entry_field_name.next; \ + (head_ptr)->head = (entry_ptr); \ + (entry_ptr)->entry_field_name.prev = &(head_ptr)->head; \ + } while (/*CONSTCOND*/ 0) + +/* TODO would be nice to not have any condition */ +#define PDC_LIST_REMOVE(entry_ptr, entry_field_name) \ + do { \ + if ((entry_ptr)->entry_field_name.next != NULL) \ + (entry_ptr)->entry_field_name.next->entry_field_name.prev = (entry_ptr)->entry_field_name.prev; \ + *(entry_ptr)->entry_field_name.prev = (entry_ptr)->entry_field_name.next; \ + } while (/*CONSTCOND*/ 0) + +#define PDC_LIST_FOREACH(var, head_ptr, entry_field_name) \ + for ((var) = ((head_ptr)->head); (var); (var) = ((var)->entry_field_name.next)) + +#define PDC_LIST_SEARCH(var, head_ptr, entry_field_name, item, value) \ + for ((var) = ((head_ptr)->head); (((var)->item != value) && (var)); \ + (var) = ((var)->entry_field_name.next)) + +#define PDC_LIST_SEARCH_CONT_NAME(var, head_ptr, entry_field_name, member, n, name) \ + for ((var) = ((head_ptr)->head); \ + ((var) && strcmp(((struct _pdc_cont_info *)((var)->member))->cont_info_pub->n, name) != 0); \ + (var) = ((var)->entry_field_name.next)) + +#endif /* PDC_LINKEDLIST_H */ diff --git a/src/commons/utils/include/pdc_malloc.h b/src/commons/utils/include/pdc_malloc.h new file mode 100644 index 000000000..e8ea1941e --- /dev/null +++ b/src/commons/utils/include/pdc_malloc.h @@ -0,0 +1,59 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#ifndef PDC_MALLOC_H +#define PDC_MALLOC_H + +#include + +/***************************************/ +/* Library-private Function Prototypes */ +/***************************************/ +/** + * Create an object + * + * \param size [IN] Size of the struct to be malloced + */ +void *PDC_malloc(size_t size); + +/** + * Create an object + * + * \param size [IN] Size of the struct to be calloced + */ +void *PDC_calloc(size_t size); + +/** + * Create an object + * + * \param mem [IN] Starting address of memory + */ +void *PDC_free(void *mem); + +#define PDC_MALLOC(t) (t *)PDC_malloc(sizeof(t)) +#define PDC_CALLOC(t) (t *)PDC_calloc(sizeof(t)) + +#define PDC_FREE(t, obj) (t *)(intptr_t) PDC_free(obj) + +#endif /* PDC_MALLOC_H */ diff --git a/src/commons/utils/include/pdc_private.h b/src/commons/utils/include/pdc_private.h new file mode 100644 index 000000000..b0fa48f14 --- /dev/null +++ b/src/commons/utils/include/pdc_private.h @@ -0,0 +1,202 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#ifndef PDC_PRIVATE_H +#define PDC_PRIVATE_H + +#include "pdc_config.h" +#include "pdc_public.h" +#include +// #include /* gettimeofday() */ + +/****************************/ +/* Library Private Typedefs */ +/****************************/ +typedef enum { + UNKNOWN = 0, + SERVER_MEMORY = 1, + CLIENT_MEMORY = 2, + FLASH = 3, + DISK = 4, + FILESYSTEM = 5, + TAPE = 6 +} _pdc_loci_t; + +/* Query type */ +typedef enum { + PDC_Q_TYPE_DATA_ELEM, /* selects data elements */ + PDC_Q_TYPE_ATTR_VALUE, /* selects attribute values */ + PDC_Q_TYPE_ATTR_NAME, /* selects attributes */ + PDC_Q_TYPE_LINK_NAME, /* selects objects */ + PDC_Q_TYPE_MISC /* (for combine queries) selects misc objects */ +} _pdc_query_type_t; + +/* Query match conditions */ +typedef enum { + PDC_Q_MATCH_EQUAL, /* equal */ + PDC_Q_MATCH_NOT_EQUAL, /* not equal */ + PDC_Q_MATCH_LESS_THAN, /* less than */ + PDC_Q_MATCH_GREATER_THAN /* greater than */ +} _pdc_query_op_t; + +typedef enum { ROW_major, COL_major } _pdc_major_type_t; + +typedef enum { C_lang = 0, FORTRAN_lang, PYTHON_lang, JULIA_lang, N_LANGUAGES } _pdc_analysis_language_t; + +/***************************/ +/* Library Private Structs */ +/***************************/ +struct _pdc_class { + char * name; + pdcid_t local_id; +}; + +#ifdef __cplusplus +#define ATTRIBUTE(a) +#else /* __cplusplus */ +#if defined(HAVE_ATTRIBUTE) +#define ATTRIBUTE(a) __attribute__((a)) +#else +#define ATTRIBUTE(a) +#endif +#endif /* __cplusplus */ + +#ifdef __cplusplus +#define ATTR_UNUSED /*void*/ +#else /* __cplusplus */ +#if defined(HAVE_ATTRIBUTE) && !defined(__SUNPRO_C) +#define ATTR_UNUSED __attribute__((unused)) +#else +#define ATTR_UNUSED /*void*/ +#endif +#endif /* __cplusplus */ + +#define PDCmemset(X, C, Z) memset((void *)(X), C, Z) + +/* + * PDC Boolean type. + */ +#ifndef FALSE +#define FALSE 0 +#endif +#ifndef TRUE +#define TRUE 1 +#endif + +extern pbool_t err_occurred; + +/* + * PGOTO_DONE macro. The argument is the return value which is + * assigned to the `ret_value' variable. Control branches to + * the `done' label. + */ +#define PGOTO_DONE(ret_val) \ + do { \ + ret_value = ret_val; \ + goto done; \ + } while (0) + +#define PGOTO_DONE_VOID \ + do { \ + goto done; \ + } while (0) + +/* + * PGOTO_ERROR macro. The arguments are the return value and an + * error string. The return value is assigned to a variable `ret_value' and + * control branches to the `done' label. + */ +#define PGOTO_ERROR(ret_val, ...) \ + do { \ + fprintf(stderr, "Error in %s:%d\n", __FILE__, __LINE__); \ + fprintf(stderr, " # %s(): ", __func__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + PGOTO_DONE(ret_val); \ + } while (0) + +#define PGOTO_ERROR_VOID(...) \ + do { \ + fprintf(stderr, "Error in %s:%d\n", __FILE__, __LINE__); \ + fprintf(stderr, " # %s(): ", __func__); \ + fprintf(stderr, "\n"); \ + PGOTO_DONE_VOID; \ + } while (0) + +/* Include a basic profiling interface */ +#ifdef ENABLE_PROFILING +#include "pdc_stack_ops.h" + +#define FUNC_ENTER(X) \ + do { \ + if (enableProfiling) \ + push(__func__, (X)); \ + } while (0) + +#define FUNC_LEAVE(ret_value) \ + do { \ + if (enableProfiling) \ + pop(); \ + return (ret_value); \ + } while (0) + +#define FUNC_LEAVE_VOID \ + do { \ + if (enableProfiling) \ + pop(); \ + return; \ + } while (0) + +#else +/* #define FUNC_ENTER(X) \ */ +/* do { \ */ +/* time_t now; \ */ +/* time(&now); \ */ +/* fprintf(stderr, "%ld enter %s\n", now, __func__); \ */ +/* } while (0) */ + +/* #define FUNC_LEAVE(ret_value) \ */ +/* do { \ */ +/* time_t now; \ */ +/* time(&now); \ */ +/* fprintf(stderr, "%ld leave %s\n", now, __func__); \ */ +/* return (ret_value); \ */ +/* } while (0) */ + +#define FUNC_ENTER(X) \ + do { \ + } while (0) + +#define FUNC_LEAVE(ret_value) \ + do { \ + return (ret_value); \ + } while (0) + +#define FUNC_LEAVE_VOID \ + do { \ + return; \ + } while (0) +#endif + +#endif /* PDC_PRIVATE_H */ diff --git a/src/commons/utils/include/pdc_timing.h b/src/commons/utils/include/pdc_timing.h new file mode 100644 index 000000000..1ea2f475b --- /dev/null +++ b/src/commons/utils/include/pdc_timing.h @@ -0,0 +1,193 @@ +#ifndef PDC_TIMING_H +#define PDC_TIMING_H + +#ifndef HOST_NAME_MAX +#if defined(__APPLE__) +#define HOST_NAME_MAX 255 +#else +#define HOST_NAME_MAX 64 +#endif /* __APPLE__ */ +#endif /* HOST_NAME_MAX */ + +#include "pdc_config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef PDC_TIMING +typedef struct pdc_timing { + double PDCbuf_obj_map_rpc; + double PDCbuf_obj_unmap_rpc; + + double PDCreg_obtain_lock_write_rpc; + double PDCreg_obtain_lock_read_rpc; + + double PDCreg_release_lock_write_rpc; + double PDCreg_release_lock_read_rpc; + + double PDCbuf_obj_map_rpc_wait; + double PDCbuf_obj_unmap_rpc_wait; + + double PDCreg_obtain_lock_write_rpc_wait; + double PDCreg_obtain_lock_read_rpc_wait; + double PDCreg_release_lock_write_rpc_wait; + double PDCreg_release_lock_read_rpc_wait; + + double PDCtransfer_request_start_write_rpc; + double PDCtransfer_request_wait_write_rpc; + double PDCtransfer_request_start_read_rpc; + double PDCtransfer_request_wait_read_rpc; + + double PDCtransfer_request_start_write_rpc_wait; + double PDCtransfer_request_start_read_rpc_wait; + double PDCtransfer_request_wait_write_rpc_wait; + double PDCtransfer_request_wait_read_rpc_wait; + + double PDCtransfer_request_start_all_write_rpc; + double PDCtransfer_request_start_all_read_rpc; + double PDCtransfer_request_wait_all_rpc; + + double PDCtransfer_request_start_all_write_rpc_wait; + double PDCtransfer_request_start_all_read_rpc_wait; + double PDCtransfer_request_wait_all_rpc_wait; + + double PDCtransfer_request_metadata_query_rpc; + + double PDCclient_obj_create_rpc; + double PDCclient_cont_create_rpc; + +} pdc_timing; + +pdc_timing pdc_timings; + +typedef struct pdc_server_timing { + double PDCbuf_obj_map_rpc; + double PDCbuf_obj_unmap_rpc; + + double PDCreg_obtain_lock_write_rpc; + double PDCreg_obtain_lock_read_rpc; + double PDCreg_release_lock_write_rpc; + double PDCreg_release_lock_read_rpc; + double PDCreg_release_lock_bulk_transfer_write_rpc; + double PDCreg_release_lock_bulk_transfer_read_rpc; + double PDCreg_release_lock_bulk_transfer_inner_write_rpc; + double PDCreg_release_lock_bulk_transfer_inner_read_rpc; + + double PDCreg_transfer_request_start_write_rpc; + double PDCreg_transfer_request_start_read_rpc; + double PDCreg_transfer_request_wait_write_rpc; + double PDCreg_transfer_request_wait_read_rpc; + double PDCreg_transfer_request_start_write_bulk_rpc; + double PDCreg_transfer_request_inner_write_bulk_rpc; + double PDCreg_transfer_request_start_read_bulk_rpc; + double PDCreg_transfer_request_inner_read_bulk_rpc; + + double PDCreg_transfer_request_start_all_write_rpc; + double PDCreg_transfer_request_start_all_read_rpc; + double PDCreg_transfer_request_start_all_write_bulk_rpc; + double PDCreg_transfer_request_start_all_read_bulk_rpc; + double PDCreg_transfer_request_inner_write_all_bulk_rpc; + double PDCreg_transfer_request_inner_read_all_bulk_rpc; + double PDCreg_transfer_request_wait_all_rpc; + double PDCreg_transfer_request_wait_all_bulk_rpc; + + double PDCdata_server_write_out; + double PDCdata_server_read_from; + double PDCcache_write; + double PDCcache_read; + double PDCcache_flush; + double PDCcache_clean; + double PDCdata_server_write_posix; + double PDCdata_server_read_posix; + + double PDCserver_obj_create_rpc; + double PDCserver_cont_create_rpc; + + double PDCserver_restart; + double PDCserver_checkpoint; + double PDCserver_start_total; +} pdc_server_timing; + +typedef struct pdc_timestamp { + double *start; + double *end; + size_t timestamp_max_size; + size_t timestamp_size; +} pdc_timestamp; + +pdc_server_timing *pdc_server_timings; +pdc_timestamp * pdc_buf_obj_map_timestamps; +pdc_timestamp * pdc_buf_obj_unmap_timestamps; + +pdc_timestamp *pdc_obtain_lock_write_timestamps; +pdc_timestamp *pdc_obtain_lock_read_timestamps; +pdc_timestamp *pdc_release_lock_write_timestamps; +pdc_timestamp *pdc_release_lock_read_timestamps; +pdc_timestamp *pdc_release_lock_bulk_transfer_write_timestamps; +pdc_timestamp *pdc_release_lock_bulk_transfer_inner_write_timestamps; +pdc_timestamp *pdc_release_lock_bulk_transfer_read_timestamps; +pdc_timestamp *pdc_release_lock_bulk_transfer_inner_read_timestamps; + +pdc_timestamp *pdc_transfer_request_start_write_timestamps; +pdc_timestamp *pdc_transfer_request_start_read_timestamps; +pdc_timestamp *pdc_transfer_request_wait_write_timestamps; +pdc_timestamp *pdc_transfer_request_wait_read_timestamps; +pdc_timestamp *pdc_transfer_request_start_write_bulk_timestamps; +pdc_timestamp *pdc_transfer_request_inner_write_bulk_timestamps; +pdc_timestamp *pdc_transfer_request_start_read_bulk_timestamps; +pdc_timestamp *pdc_transfer_request_inner_read_bulk_timestamps; + +pdc_timestamp *pdc_transfer_request_start_all_write_timestamps; +pdc_timestamp *pdc_transfer_request_start_all_read_timestamps; +pdc_timestamp *pdc_transfer_request_start_all_write_bulk_timestamps; +pdc_timestamp *pdc_transfer_request_start_all_read_bulk_timestamps; +pdc_timestamp *pdc_transfer_request_wait_all_timestamps; +pdc_timestamp *pdc_transfer_request_inner_write_all_bulk_timestamps; +pdc_timestamp *pdc_transfer_request_inner_read_all_bulk_timestamps; + +pdc_timestamp *pdc_client_buf_obj_map_timestamps; +pdc_timestamp *pdc_client_buf_obj_unmap_timestamps; +pdc_timestamp *pdc_client_obtain_lock_write_timestamps; +pdc_timestamp *pdc_client_obtain_lock_read_timestamps; +pdc_timestamp *pdc_client_release_lock_write_timestamps; +pdc_timestamp *pdc_client_release_lock_read_timestamps; + +pdc_timestamp *pdc_client_transfer_request_start_write_timestamps; +pdc_timestamp *pdc_client_transfer_request_start_read_timestamps; +pdc_timestamp *pdc_client_transfer_request_wait_write_timestamps; +pdc_timestamp *pdc_client_transfer_request_wait_read_timestamps; + +pdc_timestamp *pdc_client_transfer_request_start_all_write_timestamps; +pdc_timestamp *pdc_client_transfer_request_start_all_read_timestamps; +pdc_timestamp *pdc_client_transfer_request_wait_all_timestamps; + +pdc_timestamp *pdc_client_create_cont_timestamps; +pdc_timestamp *pdc_client_create_obj_timestamps; + +pdc_timestamp *pdc_client_transfer_request_metadata_query_timestamps; + +int PDC_timing_init(); +int PDC_timing_finalize(); +int PDC_timing_report(const char *prefix); +int PDC_server_timing_init(); +int pdc_timestamp_register(pdc_timestamp *timestamp, double start, double end); +int PDC_server_timing_report(); +#else +int PDC_timing_report(const char *prefix); +#endif + +#endif diff --git a/src/commons/utils/pdc_malloc.c b/src/commons/utils/pdc_malloc.c new file mode 100644 index 000000000..40379d400 --- /dev/null +++ b/src/commons/utils/pdc_malloc.c @@ -0,0 +1,76 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include "pdc_malloc.h" +#include "pdc_private.h" + +void * +PDC_malloc(size_t size) +{ + void *ret_value; + + FUNC_ENTER(NULL); + + assert(size); + + if (size) + ret_value = malloc(size); + else + ret_value = NULL; + + FUNC_LEAVE(ret_value); +} + +void * +PDC_calloc(size_t size) +{ + void *ret_value; + + FUNC_ENTER(NULL); + + assert(size); + + if (size) + ret_value = calloc(1, size); + else + ret_value = NULL; + + FUNC_LEAVE(ret_value); +} + +void * +PDC_free(void *mem) +{ + void *ret_value = NULL; + + FUNC_ENTER(NULL); + + if (mem) { + free(mem); + } + + FUNC_LEAVE(ret_value); +} diff --git a/src/commons/utils/pdc_timing.c b/src/commons/utils/pdc_timing.c new file mode 100644 index 000000000..95826b56b --- /dev/null +++ b/src/commons/utils/pdc_timing.c @@ -0,0 +1,537 @@ +#include "pdc_timing.h" + +#ifdef PDC_TIMING +static double pdc_base_time; + +static int +pdc_timestamp_clean(pdc_timestamp *timestamp) +{ + if (timestamp->timestamp_size) { + free(timestamp->start); + } + return 0; +} + +static int +timestamp_log(FILE *stream, const char *header, pdc_timestamp *timestamp) +{ + size_t i; + double total = 0.0; + fprintf(stream, "%s", header); + for (i = 0; i < timestamp->timestamp_size; ++i) { + fprintf(stream, ",%4f-%4f", timestamp->start[i], timestamp->end[i]); + total += timestamp->end[i] - timestamp->start[i]; + } + fprintf(stream, "\n"); + + if (i > 0) + fprintf(stream, "%s_total, %f\n", header, total); + + return 0; +} + +int +PDC_timing_init() +{ + char hostname[HOST_NAME_MAX]; + int rank; + pdc_timestamp *ptr; + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + gethostname(hostname, HOST_NAME_MAX); + if (!(rank % 31)) { + printf("client process rank %d, hostname = %s\n", rank, hostname); + } + MPI_Barrier(MPI_COMM_WORLD); + + memset(&pdc_timings, 0, sizeof(pdc_timing)); + + pdc_client_buf_obj_map_timestamps = calloc(16, sizeof(pdc_timestamp)); + ptr = pdc_client_buf_obj_map_timestamps + 1; + pdc_client_buf_obj_unmap_timestamps = ptr; + ptr++; + pdc_client_obtain_lock_write_timestamps = ptr; + ptr++; + pdc_client_obtain_lock_read_timestamps = ptr; + ptr++; + pdc_client_release_lock_write_timestamps = ptr; + ptr++; + pdc_client_release_lock_read_timestamps = ptr; + ptr++; + + pdc_client_transfer_request_start_write_timestamps = ptr; + ptr++; + pdc_client_transfer_request_start_read_timestamps = ptr; + ptr++; + pdc_client_transfer_request_wait_write_timestamps = ptr; + ptr++; + pdc_client_transfer_request_wait_read_timestamps = ptr; + ptr++; + + pdc_client_transfer_request_start_all_write_timestamps = ptr; + ptr++; + pdc_client_transfer_request_start_all_read_timestamps = ptr; + ptr++; + pdc_client_transfer_request_wait_all_timestamps = ptr; + ptr++; + + pdc_client_create_cont_timestamps = ptr; + ptr++; + pdc_client_create_obj_timestamps = ptr; + + ptr++; + pdc_client_transfer_request_metadata_query_timestamps = ptr; + + return 0; +} + +int +PDC_timing_finalize() +{ + pdc_timestamp_clean(pdc_client_buf_obj_map_timestamps); + pdc_timestamp_clean(pdc_client_buf_obj_unmap_timestamps); + + pdc_timestamp_clean(pdc_client_obtain_lock_write_timestamps); + pdc_timestamp_clean(pdc_client_obtain_lock_read_timestamps); + pdc_timestamp_clean(pdc_client_release_lock_write_timestamps); + pdc_timestamp_clean(pdc_client_release_lock_read_timestamps); + + pdc_timestamp_clean(pdc_client_transfer_request_start_write_timestamps); + pdc_timestamp_clean(pdc_client_transfer_request_start_read_timestamps); + pdc_timestamp_clean(pdc_client_transfer_request_wait_write_timestamps); + pdc_timestamp_clean(pdc_client_transfer_request_wait_read_timestamps); + pdc_timestamp_clean(pdc_client_create_cont_timestamps); + pdc_timestamp_clean(pdc_client_create_obj_timestamps); + pdc_timestamp_clean(pdc_client_transfer_request_start_all_write_timestamps); + pdc_timestamp_clean(pdc_client_transfer_request_start_all_read_timestamps); + pdc_timestamp_clean(pdc_client_transfer_request_wait_all_timestamps); + pdc_timestamp_clean(pdc_client_transfer_request_metadata_query_timestamps); + + free(pdc_client_buf_obj_map_timestamps); + return 0; +} + +int +PDC_timing_report(const char *prefix) +{ + pdc_timing max_timings; + int rank; + char filename[256], header[256]; + FILE * stream; + char hostname[HOST_NAME_MAX]; + time_t now; + + time(&now); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + gethostname(hostname, HOST_NAME_MAX); + if (!(rank % 32)) { + printf("client process rank %d, hostname = %s\n", rank, hostname); + } + MPI_Reduce(&pdc_timings, &max_timings, sizeof(pdc_timing) / sizeof(double), MPI_DOUBLE, MPI_MAX, 0, + MPI_COMM_WORLD); + if (rank == 0) { + printf("PDCbuf_obj_map_rpc = %lf, wait = %lf\n", max_timings.PDCbuf_obj_map_rpc, + max_timings.PDCbuf_obj_map_rpc_wait); + printf("PDCreg_obtain_lock_write_rpc = %lf, wait = %lf\n", max_timings.PDCreg_obtain_lock_write_rpc, + max_timings.PDCreg_obtain_lock_write_rpc_wait); + printf("PDCreg_obtain_lock_read_rpc = %lf, wait = %lf\n", max_timings.PDCreg_obtain_lock_read_rpc, + max_timings.PDCreg_obtain_lock_read_rpc_wait); + + printf("PDCreg_release_lock_write_rpc = %lf, wait = %lf\n", max_timings.PDCreg_release_lock_write_rpc, + max_timings.PDCreg_release_lock_write_rpc_wait); + printf("PDCreg_release_lock_read_rpc = %lf, wait = %lf\n", max_timings.PDCreg_release_lock_read_rpc, + max_timings.PDCreg_release_lock_read_rpc_wait); + printf("PDCbuf_obj_unmap_rpc = %lf, wait = %lf\n", max_timings.PDCbuf_obj_unmap_rpc, + max_timings.PDCbuf_obj_unmap_rpc_wait); + + printf("PDCtransfer_request_start_write = %lf, wait = %lf\n", + max_timings.PDCtransfer_request_start_write_rpc, + max_timings.PDCtransfer_request_start_write_rpc_wait); + printf("PDCtransfer_request_start_read = %lf, wait = %lf\n", + max_timings.PDCtransfer_request_start_read_rpc, + max_timings.PDCtransfer_request_start_read_rpc_wait); + printf("PDCtransfer_request_wait_write = %lf, wait = %lf\n", + max_timings.PDCtransfer_request_wait_write_rpc, + max_timings.PDCtransfer_request_wait_write_rpc_wait); + printf("PDCtransfer_request_wait_read = %lf, wait = %lf\n", + max_timings.PDCtransfer_request_wait_read_rpc, + max_timings.PDCtransfer_request_wait_read_rpc_wait); + printf("PDCtransfer_request_start_all_write = %lf, wait = %lf\n", + max_timings.PDCtransfer_request_start_all_write_rpc, + max_timings.PDCtransfer_request_start_all_write_rpc_wait); + printf("PDCtransfer_request_start_all_read = %lf, wait = %lf\n", + max_timings.PDCtransfer_request_start_all_read_rpc, + max_timings.PDCtransfer_request_start_all_read_rpc_wait); + printf("PDCtransfer_request_wait_write = %lf, wait = %lf\n", + max_timings.PDCtransfer_request_wait_all_rpc, + max_timings.PDCtransfer_request_wait_all_rpc_wait); + } + + sprintf(filename, "pdc_client_log_rank_%d.csv", rank); + stream = fopen(filename, "r"); + if (stream) { + fclose(stream); + stream = fopen(filename, "a"); + } + else { + stream = fopen(filename, "w"); + } + + fprintf(stream, "%s", ctime(&now)); + + sprintf(header, "buf_obj_map_%s", prefix); + timestamp_log(stream, header, pdc_client_buf_obj_map_timestamps); + sprintf(header, "buf_obj_unmap_%s", prefix); + timestamp_log(stream, header, pdc_client_buf_obj_unmap_timestamps); + + sprintf(header, "obtain_lock_write_%s", prefix); + timestamp_log(stream, header, pdc_client_obtain_lock_write_timestamps); + sprintf(header, "obtain_lock_read_%s", prefix); + timestamp_log(stream, header, pdc_client_obtain_lock_read_timestamps); + + sprintf(header, "release_lock_write_%s", prefix); + timestamp_log(stream, header, pdc_client_release_lock_write_timestamps); + sprintf(header, "release_lock_read_%s", prefix); + timestamp_log(stream, header, pdc_client_release_lock_read_timestamps); + + sprintf(header, "transfer_request_start_write_%s", prefix); + timestamp_log(stream, header, pdc_client_transfer_request_start_write_timestamps); + + sprintf(header, "transfer_request_start_read_%s", prefix); + timestamp_log(stream, header, pdc_client_transfer_request_start_read_timestamps); + + sprintf(header, "transfer_request_wait_write_%s", prefix); + timestamp_log(stream, header, pdc_client_transfer_request_wait_write_timestamps); + + sprintf(header, "transfer_request_wait_read_%s", prefix); + timestamp_log(stream, header, pdc_client_transfer_request_wait_read_timestamps); + + sprintf(header, "transfer_request_start_all_write_%s", prefix); + timestamp_log(stream, header, pdc_client_transfer_request_start_all_write_timestamps); + + sprintf(header, "transfer_request_start_all_read_%s", prefix); + timestamp_log(stream, header, pdc_client_transfer_request_start_all_read_timestamps); + + sprintf(header, "transfer_request_wait_all_%s", prefix); + timestamp_log(stream, header, pdc_client_transfer_request_wait_all_timestamps); + + sprintf(header, "create_cont"); + timestamp_log(stream, header, pdc_client_create_cont_timestamps); + + sprintf(header, "create_obj"); + timestamp_log(stream, header, pdc_client_create_obj_timestamps); + + fprintf(stream, "\n"); + fclose(stream); + + pdc_client_buf_obj_map_timestamps->timestamp_size = 0; + pdc_client_buf_obj_unmap_timestamps->timestamp_size = 0; + + pdc_client_obtain_lock_write_timestamps->timestamp_size = 0; + pdc_client_obtain_lock_read_timestamps->timestamp_size = 0; + pdc_client_release_lock_write_timestamps->timestamp_size = 0; + pdc_client_release_lock_read_timestamps->timestamp_size = 0; + + pdc_client_transfer_request_start_write_timestamps->timestamp_size = 0; + pdc_client_transfer_request_start_read_timestamps->timestamp_size = 0; + pdc_client_transfer_request_wait_write_timestamps->timestamp_size = 0; + pdc_client_transfer_request_wait_read_timestamps->timestamp_size = 0; + + pdc_client_transfer_request_start_all_write_timestamps->timestamp_size = 0; + pdc_client_transfer_request_start_all_read_timestamps->timestamp_size = 0; + pdc_client_transfer_request_wait_all_timestamps->timestamp_size = 0; + + pdc_client_create_cont_timestamps->timestamp_size = 0; + pdc_client_create_obj_timestamps->timestamp_size = 0; + + pdc_client_transfer_request_metadata_query_timestamps->timestamp_size = 0; + + memset(&pdc_timings, 0, sizeof(pdc_timings)); + + return 0; +} + +int +PDC_server_timing_init() +{ + char hostname[HOST_NAME_MAX]; + int rank; + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + gethostname(hostname, HOST_NAME_MAX); + + printf("server process rank %d, hostname = %s\n", rank, hostname); + /* + printf("rank = %d, hostname = %s, PDCbuf_obj_map_rpc = %lf, PDCreg_obtain_lock_rpc = %lf, " + "PDCreg_release_lock_write_rpc = " + "%lf, PDCreg_release_lock_read_rpc = %lf, PDCbuf_obj_unmap_rpc = %lf, " + "region_release_bulk_transfer_cb = %lf\n", + rank, hostname, server_timings->PDCbuf_obj_map_rpc, server_timings->PDCreg_obtain_lock_rpc, + server_timings->PDCreg_release_lock_write_rpc, server_timings->PDCreg_release_lock_read_rpc, + server_timings->PDCbuf_obj_unmap_rpc, server_timings->PDCreg_release_lock_bulk_transfer_rpc); + */ + MPI_Barrier(MPI_COMM_WORLD); + + pdc_server_timings = calloc(1, sizeof(pdc_server_timing)); + pdc_timestamp *ptr = calloc(25, sizeof(pdc_timestamp)); + pdc_buf_obj_map_timestamps = ptr; + ptr++; + pdc_buf_obj_unmap_timestamps = ptr; + ptr++; + pdc_obtain_lock_write_timestamps = ptr; + ptr++; + pdc_obtain_lock_read_timestamps = ptr; + ptr++; + pdc_release_lock_write_timestamps = ptr; + ptr++; + pdc_release_lock_read_timestamps = ptr; + ptr++; + pdc_release_lock_bulk_transfer_write_timestamps = ptr; + ptr++; + pdc_release_lock_bulk_transfer_read_timestamps = ptr; + ptr++; + pdc_release_lock_bulk_transfer_inner_write_timestamps = ptr; + ptr++; + pdc_release_lock_bulk_transfer_inner_read_timestamps = ptr; + ptr++; + + pdc_transfer_request_start_write_timestamps = ptr; + ptr++; + pdc_transfer_request_start_read_timestamps = ptr; + ptr++; + pdc_transfer_request_wait_write_timestamps = ptr; + ptr++; + pdc_transfer_request_wait_read_timestamps = ptr; + ptr++; + pdc_transfer_request_start_write_bulk_timestamps = ptr; + ptr++; + pdc_transfer_request_start_read_bulk_timestamps = ptr; + ptr++; + pdc_transfer_request_inner_write_bulk_timestamps = ptr; + ptr++; + pdc_transfer_request_inner_read_bulk_timestamps = ptr; + ptr++; + + pdc_transfer_request_start_all_write_timestamps = ptr; + ptr++; + pdc_transfer_request_start_all_read_timestamps = ptr; + ptr++; + pdc_transfer_request_wait_all_timestamps = ptr; + ptr++; + pdc_transfer_request_start_all_write_bulk_timestamps = ptr; + ptr++; + pdc_transfer_request_start_all_read_bulk_timestamps = ptr; + ptr++; + pdc_transfer_request_inner_write_all_bulk_timestamps = ptr; + ptr++; + pdc_transfer_request_inner_read_all_bulk_timestamps = ptr; + ptr++; + + // 25 timestamps + + pdc_base_time = MPI_Wtime(); + return 0; +} + +int +pdc_timestamp_register(pdc_timestamp *timestamp, double start, double end) +{ + double *temp; + + if (timestamp->timestamp_max_size == 0) { + timestamp->timestamp_max_size = 256; + timestamp->start = (double *)malloc(sizeof(double) * timestamp->timestamp_max_size * 2); + timestamp->end = timestamp->start + timestamp->timestamp_max_size; + timestamp->timestamp_size = 0; + } + else if (timestamp->timestamp_size == timestamp->timestamp_max_size) { + temp = (double *)malloc(sizeof(double) * timestamp->timestamp_max_size * 4); + memcpy(temp, timestamp->start, sizeof(double) * timestamp->timestamp_max_size); + memcpy(temp + timestamp->timestamp_max_size * 2, timestamp->end, + sizeof(double) * timestamp->timestamp_max_size); + timestamp->start = temp; + timestamp->end = temp + timestamp->timestamp_max_size * 2; + timestamp->timestamp_max_size *= 2; + } + timestamp->start[timestamp->timestamp_size] = start; + timestamp->end[timestamp->timestamp_size] = end; + timestamp->timestamp_size++; + return 0; +} + +int +PDC_server_timing_report() +{ + pdc_server_timing max_timings; + int rank; + char filename[256]; + FILE * stream; + + // char hostname[HOST_NAME_MAX]; + time_t now; + + time(&now); + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Reduce(pdc_server_timings, &max_timings, sizeof(pdc_server_timing) / sizeof(double), MPI_DOUBLE, + MPI_MAX, 0, MPI_COMM_WORLD); + sprintf(filename, "pdc_server_log_rank_%d.csv", rank); + + stream = fopen(filename, "w"); + + fprintf(stream, "%s", ctime(&now)); + timestamp_log(stream, "buf_obj_map", pdc_buf_obj_map_timestamps); + timestamp_log(stream, "buf_obj_unmap", pdc_buf_obj_unmap_timestamps); + + timestamp_log(stream, "obtain_lock_write", pdc_obtain_lock_write_timestamps); + timestamp_log(stream, "obtain_lock_read", pdc_obtain_lock_read_timestamps); + timestamp_log(stream, "release_lock_write", pdc_release_lock_write_timestamps); + timestamp_log(stream, "release_lock_read", pdc_release_lock_read_timestamps); + timestamp_log(stream, "release_lock_bulk_transfer_write", + pdc_release_lock_bulk_transfer_write_timestamps); + timestamp_log(stream, "release_lock_bulk_transfer_read", pdc_release_lock_bulk_transfer_read_timestamps); + timestamp_log(stream, "release_lock_bulk_transfer_inner_write", + pdc_release_lock_bulk_transfer_inner_write_timestamps); + timestamp_log(stream, "release_lock_bulk_transfer_inner_read", + pdc_release_lock_bulk_transfer_inner_read_timestamps); + + timestamp_log(stream, "transfer_request_start_write", pdc_transfer_request_start_write_timestamps); + timestamp_log(stream, "transfer_request_wait_write", pdc_transfer_request_wait_write_timestamps); + timestamp_log(stream, "transfer_request_start_write_bulk", + pdc_transfer_request_start_write_bulk_timestamps); + timestamp_log(stream, "transfer_request_inner_write_bulk", + pdc_transfer_request_inner_write_bulk_timestamps); + timestamp_log(stream, "transfer_request_start_read", pdc_transfer_request_start_read_timestamps); + timestamp_log(stream, "transfer_request_wait_read", pdc_transfer_request_wait_read_timestamps); + timestamp_log(stream, "transfer_request_start_read_bulk", + pdc_transfer_request_start_read_bulk_timestamps); + timestamp_log(stream, "transfer_request_inner_read_bulk", + pdc_transfer_request_inner_read_bulk_timestamps); + + timestamp_log(stream, "transfer_request_start_all_write", + pdc_transfer_request_start_all_write_timestamps); + timestamp_log(stream, "transfer_request_start_all_write_bulk", + pdc_transfer_request_start_all_write_bulk_timestamps); + timestamp_log(stream, "transfer_request_start_all_read", pdc_transfer_request_start_all_read_timestamps); + timestamp_log(stream, "transfer_request_start_all_read_bulk", + pdc_transfer_request_start_all_read_bulk_timestamps); + timestamp_log(stream, "transfer_request_inner_write_all_bulk", + pdc_transfer_request_inner_write_all_bulk_timestamps); + timestamp_log(stream, "transfer_request_inner_read_all_bulk", + pdc_transfer_request_inner_read_all_bulk_timestamps); + timestamp_log(stream, "transfer_request_wait_all", pdc_transfer_request_wait_all_timestamps); + + /* timestamp_log(stream, "create_obj", create_obj_timestamps); */ + /* timestamp_log(stream, "create_cont", create_cont_timestamps); */ + fclose(stream); + + sprintf(filename, "pdc_server_timings_%d.csv", rank); + stream = fopen(filename, "w"); + fprintf(stream, "%s", ctime(&now)); + fprintf(stream, "PDCbuf_obj_map_rpc, %lf\n", pdc_server_timings->PDCbuf_obj_map_rpc); + fprintf(stream, "PDCreg_obtain_lock_write_rpc, %lf\n", pdc_server_timings->PDCreg_obtain_lock_write_rpc); + fprintf(stream, "PDCreg_obtain_lock_read_rpc, %lf\n", pdc_server_timings->PDCreg_obtain_lock_read_rpc); + fprintf(stream, "PDCreg_release_lock_write_rpc, %lf\n", + pdc_server_timings->PDCreg_release_lock_write_rpc); + fprintf(stream, "PDCreg_release_lock_read_rpc, %lf\n", pdc_server_timings->PDCreg_release_lock_read_rpc); + fprintf(stream, "PDCbuf_obj_unmap_rpc, %lf\n", pdc_server_timings->PDCbuf_obj_unmap_rpc); + fprintf(stream, "PDCreg_release_lock_bulk_transfer_write_rpc, %lf\n", + pdc_server_timings->PDCreg_release_lock_bulk_transfer_write_rpc); + fprintf(stream, "PDCreg_release_lock_bulk_transfer_read_rpc, %lf\n", + pdc_server_timings->PDCreg_release_lock_bulk_transfer_read_rpc); + fprintf(stream, "PDCreg_release_lock_bulk_transfer_inner_write_rpc, %lf\n", + pdc_server_timings->PDCreg_release_lock_bulk_transfer_inner_write_rpc); + fprintf(stream, "PDCreg_release_lock_bulk_transfer_inner_read_rpc, %lf\n", + pdc_server_timings->PDCreg_release_lock_bulk_transfer_inner_read_rpc); + fprintf(stream, "PDCregion_transfer_start_write_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_start_write_rpc); + fprintf(stream, "PDCregion_transfer_wait_write_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_wait_write_rpc); + fprintf(stream, "PDCregion_transfer_start_write_bulk_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_start_write_bulk_rpc); + fprintf(stream, "PDCregion_transfer_request_inner_write_bulk_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_inner_write_bulk_rpc); + fprintf(stream, "PDCregion_transfer_start_read_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_start_read_rpc); + fprintf(stream, "PDCregion_transfer_wait_read_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_wait_read_rpc); + fprintf(stream, "PDCregion_transfer_start_read_bulk_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_start_read_bulk_rpc); + fprintf(stream, "PDCregion_transfer_request_inner_read_bulk_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_inner_read_bulk_rpc); + + fprintf(stream, "PDCregion_transfer_start_write_all_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_start_all_write_rpc); + fprintf(stream, "PDCregion_transfer_request_inner_write_all_bulk_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_inner_write_all_bulk_rpc); + fprintf(stream, "PDCregion_transfer_start_all_read_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_start_all_read_rpc); + fprintf(stream, "PDCregion_transfer_request_inner_read_all_bulk_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_inner_read_all_bulk_rpc); + fprintf(stream, "PDCregion_transfer_wait_all_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_wait_all_rpc); + fprintf(stream, "PDCregion_transfer_wait_all_bulk_rpc, %lf\n", + pdc_server_timings->PDCreg_transfer_request_wait_all_bulk_rpc); + + fprintf(stream, "PDCserver_obj_create_rpc, %lf\n", pdc_server_timings->PDCserver_obj_create_rpc); + fprintf(stream, "PDCserver_cont_create_rpc, %lf\n", pdc_server_timings->PDCserver_cont_create_rpc); + + fprintf(stream, "PDCdata_server_write_out, %lf\n", pdc_server_timings->PDCdata_server_write_out); + fprintf(stream, "PDCdata_server_read_from, %lf\n", pdc_server_timings->PDCdata_server_read_from); + fprintf(stream, "PDCcache_write, %lf\n", pdc_server_timings->PDCcache_write); + fprintf(stream, "PDCcache_read, %lf\n", pdc_server_timings->PDCcache_read); + fprintf(stream, "PDCcache_flush, %lf\n", pdc_server_timings->PDCcache_flush); + fprintf(stream, "PDCcache_clean, %lf\n", pdc_server_timings->PDCcache_clean); + fprintf(stream, "PDCdata_server_write_posix, %lf\n", pdc_server_timings->PDCdata_server_write_posix); + fprintf(stream, "PDCdata_server_read_posix, %lf\n", pdc_server_timings->PDCdata_server_read_posix); + + fprintf(stream, "PDCserver_restart, %lf\n", pdc_server_timings->PDCserver_restart); + fprintf(stream, "PDCserver_checkpoint, %lf\n", pdc_server_timings->PDCserver_checkpoint); + fprintf(stream, "PDCstart_server_total, %lf\n", pdc_server_timings->PDCserver_start_total); + + fclose(stream); + + free(pdc_server_timings); + pdc_timestamp_clean(pdc_buf_obj_map_timestamps); + pdc_timestamp_clean(pdc_buf_obj_unmap_timestamps); + + pdc_timestamp_clean(pdc_obtain_lock_write_timestamps); + pdc_timestamp_clean(pdc_obtain_lock_read_timestamps); + pdc_timestamp_clean(pdc_release_lock_write_timestamps); + pdc_timestamp_clean(pdc_release_lock_read_timestamps); + pdc_timestamp_clean(pdc_release_lock_bulk_transfer_write_timestamps); + pdc_timestamp_clean(pdc_release_lock_bulk_transfer_read_timestamps); + pdc_timestamp_clean(pdc_release_lock_bulk_transfer_inner_write_timestamps); + pdc_timestamp_clean(pdc_release_lock_bulk_transfer_inner_read_timestamps); + + pdc_timestamp_clean(pdc_transfer_request_start_write_timestamps); + pdc_timestamp_clean(pdc_transfer_request_start_read_timestamps); + pdc_timestamp_clean(pdc_transfer_request_wait_write_timestamps); + pdc_timestamp_clean(pdc_transfer_request_wait_read_timestamps); + pdc_timestamp_clean(pdc_transfer_request_start_write_bulk_timestamps); + pdc_timestamp_clean(pdc_transfer_request_start_read_bulk_timestamps); + pdc_timestamp_clean(pdc_transfer_request_inner_write_bulk_timestamps); + pdc_timestamp_clean(pdc_transfer_request_inner_read_bulk_timestamps); + + pdc_timestamp_clean(pdc_transfer_request_start_all_write_timestamps); + pdc_timestamp_clean(pdc_transfer_request_start_all_read_timestamps); + pdc_timestamp_clean(pdc_transfer_request_start_all_write_bulk_timestamps); + pdc_timestamp_clean(pdc_transfer_request_start_all_read_bulk_timestamps); + pdc_timestamp_clean(pdc_transfer_request_wait_all_timestamps); + pdc_timestamp_clean(pdc_transfer_request_inner_write_all_bulk_timestamps); + pdc_timestamp_clean(pdc_transfer_request_inner_read_all_bulk_timestamps); + + /* pdc_timestamp_clean(pdc_create_obj_timestamps); */ + /* pdc_timestamp_clean(pdc_create_cont_timestamps); */ + + free(pdc_buf_obj_map_timestamps); + return 0; +} + +#else +int +PDC_timing_report(const char *prefix __attribute__((unused))) +{ + return 0; +} +#endif diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index dbbff7bfd..859ce7bb8 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -6846,10 +6846,10 @@ PDC_kvtag_dup(pdc_kvtag_t *from, pdc_kvtag_t **to) if (from == NULL || to == NULL) PGOTO_DONE(FAIL); - (*to) = (pdc_kvtag_t *)calloc(1, sizeof(pdc_kvtag_t)); - (*to)->name = (char *)malloc(strlen(from->name) + 1); - (*to)->size = from->size; - (*to)->type = from->type; + (*to) = (pdc_kvtag_t *)calloc(1, sizeof(pdc_kvtag_t)); + (*to)->name = (char *)malloc(strlen(from->name) + 1); + (*to)->size = from->size; + (*to)->type = from->type; (*to)->value = (void *)malloc(from->size); memcpy((void *)(*to)->name, (void *)from->name, strlen(from->name) + 1); memcpy((void *)(*to)->value, (void *)from->value, from->size); diff --git a/src/tests/cont_tags.c b/src/tests/cont_tags.c index f6c961a71..caaf87b49 100644 --- a/src/tests/cont_tags.c +++ b/src/tests/cont_tags.c @@ -36,9 +36,9 @@ main(int argc, char **argv) int rank = 0, size = 1; - char tag_value[128], tag_value2[128], *tag_value_ret; + char tag_value[128], tag_value2[128], *tag_value_ret; pdc_var_type_t value_type; - psize_t value_size; + psize_t value_size; strcpy(tag_value, "some tag value"); strcpy(tag_value2, "some tag value 2 is longer than tag 1"); diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index 2ccc67828..a682e30ee 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -220,7 +220,8 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value v = i + my_obj_s; for (j = 0; j < n_attr; j++) { sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", v, j); - if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], PDC_STRING, tag_value_len + 1) < 0) + if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], PDC_STRING, tag_value_len + 1) < + 0) printf("fail to add a kvtag to o%" PRIu64 "\n", v); } } @@ -238,8 +239,8 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value void get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, uint64_t n_attr, void **tag_values, uint64_t *value_size) { - uint64_t i; - char tag_name[256]; + uint64_t i; + char tag_name[256]; pdc_var_type_t tag_type; for (i = 0; i < n_attr; i++) { diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index d145560d0..dc14c597e 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -210,7 +210,8 @@ main(int argc, char *argv[]) stime = MPI_Wtime(); #endif for (i = 0; i < my_query; i++) { - if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, + (void *)&value_size) < 0) printf("fail to get a kvtag from o%d\n", i + my_query_s); if (i % query_1percent == 0) { diff --git a/src/tests/obj_tags.c b/src/tests/obj_tags.c index db99bec1b..5906e3dd9 100644 --- a/src/tests/obj_tags.c +++ b/src/tests/obj_tags.c @@ -42,10 +42,10 @@ main(int argc, char **argv) dims[0] = 64; dims[1] = 3; dims[2] = 4; - char tag_value[128], tag_value2[128], *tag_value_ret; - char cont_name[128], obj_name1[128], obj_name2[128]; + char tag_value[128], tag_value2[128], *tag_value_ret; + char cont_name[128], obj_name1[128], obj_name2[128]; pdc_var_type_t value_type; - psize_t value_size; + psize_t value_size; strcpy(tag_value, "some tag value"); strcpy(tag_value2, "some tag value 2 is longer"); diff --git a/src/tests/read_obj_shared.c b/src/tests/read_obj_shared.c index 4b0058dbd..6f56965d8 100644 --- a/src/tests/read_obj_shared.c +++ b/src/tests/read_obj_shared.c @@ -163,7 +163,7 @@ main(int argc, char **argv) offset[0] = rank * my_data_size; local_offset[0] = 0; mysize[0] = my_data_size; - printf("rank %d offset = %lu, length = %lu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); + printf("rank %d offset = %llu, length = %llu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); local_region = PDCregion_create(ndim, local_offset, mysize); global_region = PDCregion_create(ndim, offset, mysize); @@ -220,7 +220,7 @@ main(int argc, char **argv) offset[0] = rank * my_data_size; local_offset[0] = 0; mysize[0] = my_data_size; - printf("rank %d offset = %lu, length = %lu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); + printf("rank %d offset = %llu, length = %llu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); local_region = PDCregion_create(ndim, local_offset, mysize); global_region = PDCregion_create(ndim, offset, mysize); diff --git a/src/tests/vpicio_mts.c b/src/tests/vpicio_mts.c index eec66215b..65b7b4dc8 100644 --- a/src/tests/vpicio_mts.c +++ b/src/tests/vpicio_mts.c @@ -199,7 +199,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("\n#Step %d\n", i); + printf("\n#Step %llu\n", i); } #endif PDCprop_set_obj_time_step(obj_prop_xx, i); From a8cb2aaa39060e754762ca71338a58541893b12e Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Wed, 21 Jun 2023 18:23:56 -0400 Subject: [PATCH 195/216] Data type for new kvtag (including refactoring and serde framework) (#90) * remove unnecessary install block from CMakeLists.txt * update output * Revert "update output" This reverts commit fe1f8b44995bc0dabd3b957e1032c2da26f56fdd. * build kvtag_add_get_scale * comment off free * update code * 1. kvtag_scale_add_get added \n 2. uint64_t support for obj/tag/query count \n 3. moving work assigning block downwards right before creating objects \n 4. everything is tested working * do while loop added, tested with 1m object and works * 1m objects test works, 10m object test fail as the original also fails * add new executable to test set * enlarge PDC_SERVER_ID_INTERVAL * update code * update console args * add p search test * add console arg for changing number of attributes per object * free allocated memory * fix query count issue * fix attr length definition * code refactored * code refactored * code refactored * code refactored * code refactored * code refactored * fix data type * fix data type * fix data type * add client side statistics * add client side statistics * fix format * clang formatter * update CMake * update CMake * update CMake * free allocated memory properly * clang format * clang format * clang-format-10 * change file name * address review comments * update llsm importer * update llsm importer * update server checkpoint intervals * update gitignore * adding job scripts * adding one debugging msg * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update output for uint64_t * add scripts * update output for uint64_t * update output for uint64_t * update output for uint64_t * update scripts * update scripts * delete debugging message * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * update tag names * update tag names * update query startingpos * update query startingpos * update job scripts * add progressive timing for kvtag_add_get_scale * fix iteration count in final report * update job scripts and benckmark program * update message format * update message format * update message format * update message format * clang format * update job scripts * comment off object/container close procedure in benchmark to save node hours * change the max number of object to 1M * change the max length of attribute value * change the max length of attribute value * llsm tiff import test * llsm tiff import test * llsm tiff import test * llsm tiff import test * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update cmake and llsm_importer * update cmake and llsm_importer * close if in cmake * cmake fix tiff * cmake policy to suppress warning * add pdc include dir * update code * update code * update code * update code * update code * update code * update array generating method * update array generating method * update array generating method * update array generating method * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * fix return type * fix return type * add timing * add timing * fix output * llsm tiff importer 1st version: read csv and import tiff files to PDC, adding metadata available in CSV files and TIFF loader * fix vairable name * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * add scripts * add scripts * add scripts * debugging for nonMPI program * debugging for nonMPI program * debugging for nonMPI program * clang format, without PDC, everything works perfectly. program fails at PDC init stage where PDCprop_create(PDC_CONT_CREATE, pdc) is being created * enable MPI * enable MPI * enlarge BCase size * enlarge BCase size * enlarge BCase size * resolve bcast count * llsm data path in script * llsm data path in script * update csv reader * update csv reader * update csv reader * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * enlarge max write * update pdc * update pdc * update pdc * update pdc * update pdc_import.c * update pdc_import.c * update pdc_export.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update tools/cmake * clang format * clang format * added a tutorial for llsm_importer * added a tutorial for llsm_importer * make sure the line feed is included for string attribute * update timing for overall completion time * update formatting * llsm_importer (#1) formatter on llsm_importer * add type for kvtag structure (#2) * upate metadata type system * update serde framework to coupe with the new data type system * replace unnecessary data types * adding type for pdc_kvtag_t, all occurances are fixed * update new commons CMake for publishing commons * commons compilation passed * compiled * Feature/metadata type (#3) * upate metadata type system * update serde framework to coupe with the new data type system * replace unnecessary data types * adding type for pdc_kvtag_t, all occurances are fixed * update new commons CMake for publishing commons * commons compilation passed * compiled * remove unnecessary header files from installation * resolve conflict * add important files * clang formatting * update cmake --------- Co-authored-by: Houjun Tang --- CMakeLists.txt | 3 +- docs/readme.md | 154 ++++---- docs/source/api.rst | 37 +- src/api/CMakeLists.txt | 29 +- src/api/pdc_client_connect.c | 20 +- src/api/pdc_obj/include/pdc_cont.h | 6 +- src/api/pdc_obj/include/pdc_obj.h | 6 +- src/api/pdc_obj/include/pdc_prop_pkg.h | 7 +- src/api/pdc_obj/pdc_dt_conv.c | 8 +- src/api/pdc_obj/pdc_obj.c | 1 + src/commons/CMakeLists.txt | 210 +++++++++++ src/commons/generic/include/pdc_generic.h | 219 +++++++++++ src/{api => commons}/include/pdc_public.h | 20 +- src/{api => commons}/profiling/CMakeLists.txt | 9 +- .../profiling/include/pdc_hashtab.h | 0 .../profiling/include/pdc_stack_ops.h | 0 src/{api => commons}/profiling/pdc_hashtab.c | 0 .../profiling/pdc_stack_ops.c | 0 src/commons/serde/include/pdc_serde.h | 156 ++++++++ src/commons/serde/pdc_serde.c | 346 ++++++++++++++++++ src/{ => commons}/utils/include/pdc_id_pkg.h | 0 .../utils/include/pdc_linkedlist.h | 4 +- src/{ => commons}/utils/include/pdc_malloc.h | 0 src/{ => commons}/utils/include/pdc_private.h | 2 +- src/{ => commons}/utils/include/pdc_timing.h | 0 src/{ => commons}/utils/pdc_malloc.c | 0 src/{ => commons}/utils/pdc_timing.c | 0 src/server/CMakeLists.txt | 5 +- src/server/include/pdc_client_server_common.h | 5 + src/server/pdc_client_server_common.c | 12 +- src/server/pdc_server.c | 7 +- .../pdc_server_region_request_handler.h | 4 +- src/tests/cont_tags.c | 21 +- src/tests/kvtag_add_get.c | 35 +- src/tests/kvtag_add_get_benchmark.c | 10 +- src/tests/kvtag_add_get_scale.c | 9 +- src/tests/kvtag_get.c | 15 +- src/tests/kvtag_query.c | 11 +- src/tests/kvtag_query_scale.c | 4 +- src/tests/obj_tags.c | 23 +- src/tests/pdc_transforms_lib.c | 35 +- src/tests/read_obj_shared.c | 4 +- src/tests/vpicio_mts.c | 2 +- src/utils/pdc_interface.c | 2 + tools/pdc_export.c | 41 +-- tools/pdc_import.c | 9 +- tools/pdc_ls.c | 40 +- 47 files changed, 1235 insertions(+), 296 deletions(-) create mode 100644 src/commons/CMakeLists.txt create mode 100644 src/commons/generic/include/pdc_generic.h rename src/{api => commons}/include/pdc_public.h (66%) rename src/{api => commons}/profiling/CMakeLists.txt (92%) rename src/{api => commons}/profiling/include/pdc_hashtab.h (100%) rename src/{api => commons}/profiling/include/pdc_stack_ops.h (100%) rename src/{api => commons}/profiling/pdc_hashtab.c (100%) rename src/{api => commons}/profiling/pdc_stack_ops.c (100%) create mode 100644 src/commons/serde/include/pdc_serde.h create mode 100644 src/commons/serde/pdc_serde.c rename src/{ => commons}/utils/include/pdc_id_pkg.h (100%) rename src/{ => commons}/utils/include/pdc_linkedlist.h (99%) rename src/{ => commons}/utils/include/pdc_malloc.h (100%) rename src/{ => commons}/utils/include/pdc_private.h (99%) rename src/{ => commons}/utils/include/pdc_timing.h (100%) rename src/{ => commons}/utils/pdc_malloc.c (100%) rename src/{ => commons}/utils/pdc_timing.c (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e353dbc2..5e7b65d94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -257,7 +257,7 @@ endif() option(PDC_ENABLE_LUSTRE "Enable Lustre." OFF) if(PDC_ENABLE_LUSTRE) set(ENABLE_LUSTRE 1) - set(PDC_LUSTRE_TOTAL_OST "248" CACHE STRING "Number of Lustre OSTs") + set(PDC_LUSTRE_TOTAL_OST "256" CACHE STRING "Number of Lustre OSTs") endif() #----------------------------------------------------------------------------- @@ -384,6 +384,7 @@ configure_file( #----------------------------------------------------------------------------- # Source #----------------------------------------------------------------------------- +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/commons) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/api) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/server) diff --git a/docs/readme.md b/docs/readme.md index df19eba94..74be2d0e4 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,58 +1,67 @@ # PDC Documentations - + [PDC user APIs](#pdc-user-apis) - - [PDC general APIs](#pdc-general-apis) - - [PDC container APIs](#pdc-container-apis) - - [PDC object APIs](#pdc-object-apis) - - [PDC region APIs](#pdc-region-apis) - - [PDC property APIs](#pdc-property-apis) - - [PDC query APIs](#pdc-query-apis) - + [PDC data types](#PDC-type-categories) - - [Basic types](#basic-types) - - [Histogram structure](#histogram-structure) - - [Container info](#container-info) - - [Container life time](#container-life-time) - - [Object property](#object-property) - - [Object info](#object-info) - - [Object structure](#object-structure) - - [Region info](#region-info) - - [Access type](#access-type) - - [Transfer request status](#transfer-request-status) - - [Query operators](#query-operators) - - [Query structures](#query-structures) - - [Selection structure](#selection-structure) - + [Developers notes](#developers-notes) - - [How to implement an RPC from client to server](#how-to-implement-an-rpc-from-client-to-server) - - [PDC Server metadata overview](#pdc-server-metadata-overview) - + [PDC metadata structure](#pdc-metadata-structure) - + [Metadata operations at client side](#metadata-operations-at-client-side) - - [PDC metadata management strategy](#pdc-metadata-management-strategy) - + [Managing metadata and data by the same server](#managing-metadata-and-data-by-the-same-server) - + [Separate metadata server from data server](#separate-metadata-server-from-data-server) - + [Static object region mappings](#static-object-region-mappings) - + [Dynamic object region mappings](#dynamic-object-region-mappings) - - [PDC metadata management implementation](#pdc-metadata-management-implementation) - + [Create metadata](#create-metadata) - + [Binding metadata to object](#binding-metadata-to-object) - + [Register object metadata at metadata server](#register-object-metadata-at-metadata-server) - + [Retrieve metadata from metadata server](#retrieve-metadata-from-metadata-server) - + [Object metadata at client](#object-metadata-at-client) - + [Metadata at data server](#metadata-at-data-server) - + [Object metadata update](#object-metadata-update) - + [Object region metadata](#object-region-metadata) - + [Metadata checkpoint](#object-metadata-update) - - [Region transfer request at client](#region-transfer-request-at-client) - + [Region transfer request create and close](#region-transfer-request-create-and-close) - + [Region transfer request start](#region-transfer-request-start) - + [Region transfer request wait](#region-transfer-request-wait) - - [Region transfer request at server](#region-transfer-request-at-server) - + [Server region transfer request RPC](#server-region-transfer-request-rpc) - - [Server nonblocking control](#server-nonblocking-control) - - [Server region transfer request start](#server-region-transfer-request-start) - - [Server region transfer request wait](#server-region-transfer-request-wait) - + [Server region storage](#server-region-storage) - - [Storage by file offset](#storage-by-file-offset) - - [Storage by region](#storage-by-region) - - [Open tasks for PDC](#open-tasks-for-pdc) +- [PDC Documentations](#pdc-documentations) +- [PDC user APIs](#pdc-user-apis) + - [PDC general APIs](#pdc-general-apis) + - [PDC container APIs](#pdc-container-apis) + - [PDC object APIs](#pdc-object-apis) + - [PDC region APIs](#pdc-region-apis) + - [PDC property APIs](#pdc-property-apis) + - [PDC query APIs](#pdc-query-apis) + - [PDC hist APIs](#pdc-hist-apis) +- [PDC Data types](#pdc-data-types) + - [Basic types](#basic-types) + - [region transfer partition type](#region-transfer-partition-type) + - [Object consistency semantics type](#object-consistency-semantics-type) + - [Histogram structure](#histogram-structure) + - [Container info](#container-info) + - [Container life time](#container-life-time) + - [Object property public](#object-property-public) + - [Object property](#object-property) + - [Object info](#object-info) + - [Object structure](#object-structure) + - [Region info](#region-info) + - [Access type](#access-type) + - [Transfer request status](#transfer-request-status) + - [Query operators](#query-operators) + - [Query structures](#query-structures) + - [Selection structure](#selection-structure) +- [Developers notes](#developers-notes) + - [How to implement an RPC from client to server](#how-to-implement-an-rpc-from-client-to-server) + - [PDC Server metadata overview](#pdc-server-metadata-overview) + - [PDC metadata structure](#pdc-metadata-structure) + - [Metadata operations at client side](#metadata-operations-at-client-side) + - [PDC metadata management strategy](#pdc-metadata-management-strategy) + - [Managing metadata and data by the same server](#managing-metadata-and-data-by-the-same-server) + - [Separate metadata server from data server](#separate-metadata-server-from-data-server) + - [Static object region mappings](#static-object-region-mappings) + - [Dynamic object region mappings](#dynamic-object-region-mappings) + - [PDC metadata management implementation](#pdc-metadata-management-implementation) + - [Create metadata](#create-metadata) + - [Binding metadata to object](#binding-metadata-to-object) + - [Register object metadata at metadata server](#register-object-metadata-at-metadata-server) + - [Retrieve metadata from metadata server](#retrieve-metadata-from-metadata-server) + - [Object metadata at client](#object-metadata-at-client) + - [Metadata at data server](#metadata-at-data-server) + - [Object metadata update](#object-metadata-update) + - [Object region metadata](#object-region-metadata) + - [Metadata checkpoint](#metadata-checkpoint) + - [Region transfer request at client](#region-transfer-request-at-client) + - [Region transfer request create and close](#region-transfer-request-create-and-close) + - [Region transfer request start](#region-transfer-request-start) + - [Region transfer request wait](#region-transfer-request-wait) + - [Region transfer request at server](#region-transfer-request-at-server) + - [Server region transfer request RPC](#server-region-transfer-request-rpc) + - [Server nonblocking control](#server-nonblocking-control) + - [Server region transfer request start](#server-region-transfer-request-start) + - [Server region transfer request wait](#server-region-transfer-request-wait) + - [Server region storage](#server-region-storage) + - [Storage by file offset](#storage-by-file-offset) + - [Storage by region](#storage-by-region) + - [Open tasks for PDC](#open-tasks-for-pdc) + - [Replacing individual modules with efficient Hash table data structures](#replacing-individual-modules-with-efficient-hash-table-data-structures) + - [Restarting pdc\_server.exe with different numbers of servers](#restarting-pdc_serverexe-with-different-numbers-of-servers) + - [Fast region search mechanisms](#fast-region-search-mechanisms) + - [Merge overlapping regions](#merge-overlapping-regions) # PDC user APIs ## PDC general APIs + pdcid_t PDCinit(const char *pdc_name) @@ -683,21 +692,28 @@ ## Basic types ``` typedef enum { - PDC_UNKNOWN = -1, /* error */ - PDC_INT = 0, /* integer types */ - PDC_FLOAT = 1, /* floating-point types */ - PDC_DOUBLE = 2, /* double types */ - PDC_CHAR = 3, /* character types */ - PDC_COMPOUND = 4, /* compound types */ - PDC_ENUM = 5, /* enumeration types */ - PDC_ARRAY = 6, /* Array types */ - PDC_UINT = 7, /* unsigned integer types */ - PDC_INT64 = 8, /* 64-bit integer types */ - PDC_UINT64 = 9, /* 64-bit unsigned integer types */ - PDC_INT16 = 10, - PDC_INT8 = 11, - NCLASSES = 12 /* this must be last */ - } pdc_var_type_t; + PDC_UNKNOWN = -1, /* error */ + PDC_INT = 0, /* integer types (identical to int32_t) */ + PDC_FLOAT = 1, /* floating-point types */ + PDC_DOUBLE = 2, /* double types */ + PDC_CHAR = 3, /* character types */ + PDC_STRING = 4, /* string types */ + PDC_BOOLEAN = 5, /* boolean types */ + PDC_SHORT = 6, /* short types */ + PDC_UINT = 7, /* unsigned integer types (identical to uint32_t) */ + PDC_INT64 = 8, /* 64-bit integer types */ + PDC_UINT64 = 9, /* 64-bit unsigned integer types */ + PDC_INT16 = 10, /* 16-bit integer types */ + PDC_INT8 = 11, /* 8-bit integer types */ + PDC_UINT8 = 12, /* 8-bit unsigned integer types */ + PDC_UINT16 = 13, /* 16-bit unsigned integer types */ + PDC_INT32 = 14, /* 32-bit integer types */ + PDC_UINT32 = 15, /* 32-bit unsigned integer types */ + PDC_LONG = 16, /* long types */ + PDC_VOID_PTR = 17, /* void pointer type */ + PDC_SIZE_T = 18, /* size_t type */ + PDC_TYPE_COUNT = 19 /* this is the number of var types and has to be the last */ + } pdc_c_var_type_t; ``` ## region transfer partition type ``` diff --git a/docs/source/api.rst b/docs/source/api.rst index e9b1e6567..ab058f10a 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -471,21 +471,28 @@ Basic types .. code-block:: c typedef enum { - PDC_UNKNOWN = -1, /* error */ - PDC_INT = 0, /* integer types */ - PDC_FLOAT = 1, /* floating-point types */ - PDC_DOUBLE = 2, /* double types */ - PDC_CHAR = 3, /* character types */ - PDC_COMPOUND = 4, /* compound types */ - PDC_ENUM = 5, /* enumeration types */ - PDC_ARRAY = 6, /* Array types */ - PDC_UINT = 7, /* unsigned integer types */ - PDC_INT64 = 8, /* 64-bit integer types */ - PDC_UINT64 = 9, /* 64-bit unsigned integer types */ - PDC_INT16 = 10, - PDC_INT8 = 11, - NCLASSES = 12 /* this must be last */ - } pdc_var_type_t; + PDC_UNKNOWN = -1, /* error */ + PDC_INT = 0, /* integer types (identical to int32_t) */ + PDC_FLOAT = 1, /* floating-point types */ + PDC_DOUBLE = 2, /* double types */ + PDC_CHAR = 3, /* character types */ + PDC_STRING = 4, /* string types */ + PDC_BOOLEAN = 5, /* boolean types */ + PDC_SHORT = 6, /* short types */ + PDC_UINT = 7, /* unsigned integer types (identical to uint32_t) */ + PDC_INT64 = 8, /* 64-bit integer types */ + PDC_UINT64 = 9, /* 64-bit unsigned integer types */ + PDC_INT16 = 10, /* 16-bit integer types */ + PDC_INT8 = 11, /* 8-bit integer types */ + PDC_UINT8 = 12, /* 8-bit unsigned integer types */ + PDC_UINT16 = 13, /* 16-bit unsigned integer types */ + PDC_INT32 = 14, /* 32-bit integer types */ + PDC_UINT32 = 15, /* 32-bit unsigned integer types */ + PDC_LONG = 16, /* long types */ + PDC_VOID_PTR = 17, /* void pointer type */ + PDC_SIZE_T = 18, /* size_t type */ + PDC_TYPE_COUNT = 19 /* this is the number of var types and has to be the last */ + } pdc_c_var_type_t; diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index 6d4e0b722..1ba4b8389 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -1,7 +1,7 @@ #------------------------------------------------------------------------------ # Include source and build directories #------------------------------------------------------------------------------ -set( LOCAL_INCLUDE_DIR +set(LOCAL_INCLUDE_DIR ${PDC_INCLUDES_BUILD_TIME} ${PROJECT_BINARY_DIR} ${PDC_SOURCE_DIR} @@ -32,9 +32,9 @@ include_directories( # External dependencies #------------------------------------------------------------------------------ # profiling -#set(PDC_EXT_LIB_DEPENDENCIES pdcprof ${PDC_EXT_LIB_DEPENDENCIES}) -set(PDC_EXT_INCLUDE_DEPENDENCIES ${CMAKE_CURRENT_SOURCE_DIR}/profiling) -set(PDC_EXPORTED_LIBS pdcprof) +# set(PDC_EXT_LIB_DEPENDENCIES pdcprof ${PDC_EXT_LIB_DEPENDENCIES}) +# set(PDC_EXT_INCLUDE_DEPENDENCIES ${CMAKE_CURRENT_SOURCE_DIR}/profiling) +# set(PDC_EXPORTED_LIBS pdcprof) # Mercury find_package(MERCURY REQUIRED) @@ -87,22 +87,24 @@ set(PDC_SRCS ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_cache.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c - ${PDC_SOURCE_DIR}/src/utils/pdc_timing.c - ${PDC_SOURCE_DIR}/src/utils/pdc_malloc.c ${PDC_SOURCE_DIR}/src/utils/pdc_interface.c ${PDC_SOURCE_DIR}/src/utils/pdc_region_utils.c ) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/profiling) + set(PDC_COMMON_INCLUDE_DIRS ${PDC_COMMON_INCLUDE_DIRS}) + # add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/profiling) #------------------------------------------------------------------------------ # Libraries #------------------------------------------------------------------------------ # PDC set(PDC_BUILD_INCLUDE_DEPENDENCIES + ${PDC_COMMON_INCLUDE_DIRS} ${LOCAL_INCLUDE_DIR} ) +message(STATUS "PDC_BUILD_INCLUDE_DEPENDENCIES: ${PDC_BUILD_INCLUDE_DEPENDENCIES}") + add_library(pdc ${PDC_SRCS}) target_include_directories(pdc @@ -110,7 +112,12 @@ target_include_directories(pdc $ ) +message(STATUS "PDC_EXPORTED_LIBS: ${PDC_EXPORTED_LIBS}") +message(STATUS "PDC_EXT_LIB_DEPENDENCIES: ${PDC_EXT_LIB_DEPENDENCIES}") +message(STATUS "PDC_COMMONS_LIBRARIES: ${PDC_COMMONS_LIBRARIES}") + target_link_libraries(pdc + ${PDC_COMMONS_LIBRARIES} ${PDC_EXPORTED_LIBS} ${PDC_EXT_LIB_DEPENDENCIES} -ldl @@ -122,7 +129,7 @@ set(PDC_EXPORTED_LIBS pdc ${PDC_EXPORTED_LIBS}) add_executable(close_server close_server.c ) -target_link_libraries(close_server pdc) +target_link_libraries(close_server pdc ${PDC_COMMON_LIBRARIES}) install( TARGETS @@ -135,7 +142,6 @@ install( #----------------------------------------------------------------------------- set(PDC_HEADERS ${PDC_SOURCE_DIR}/src/api/include/pdc.h - ${PDC_SOURCE_DIR}/src/api/include/pdc_public.h ${PDC_SOURCE_DIR}/src/api/pdc_analysis/include/pdc_analysis.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_cont.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_mpi.h @@ -173,9 +179,10 @@ install( #----------------------------------------------------------------------------- # Add Target(s) to CMake Install #----------------------------------------------------------------------------- + install( TARGETS - pdc + pdc EXPORT ${PDC_EXPORTED_TARGETS} LIBRARY DESTINATION ${PDC_INSTALL_LIB_DIR} @@ -223,4 +230,4 @@ set(PDC_INCLUDES_INSTALL_TIME ${PDC_INSTALL_INCLUDE_DIR} ${PDC_EXT_INCLUDE_DEPENDENCIES} PARENT_SCOPE -) +) \ No newline at end of file diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index bdf967885..fff47f027 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -33,6 +33,7 @@ #include "pdc_utlist.h" #include "pdc_id_pkg.h" +#include "pdc_cont_pkg.h" #include "pdc_prop_pkg.h" #include "pdc_obj_pkg.h" #include "pdc_cont.h" @@ -7018,6 +7019,7 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) if (kvtag != NULL && kvtag != NULL && kvtag->size != 0) { in.kvtag.name = kvtag->name; in.kvtag.value = kvtag->value; + in.kvtag.type = kvtag->type; in.kvtag.size = kvtag->size; } else @@ -7061,6 +7063,7 @@ metadata_get_kvtag_rpc_cb(const struct hg_cb_info *callback_info) client_lookup_args->ret = output.ret; client_lookup_args->kvtag->name = strdup(output.kvtag.name); client_lookup_args->kvtag->size = output.kvtag.size; + client_lookup_args->kvtag->type = output.kvtag.type; client_lookup_args->kvtag->value = malloc(output.kvtag.size); memcpy(client_lookup_args->kvtag->value, output.kvtag.value, output.kvtag.size); /* PDC_kvtag_dup(&(output.kvtag), &client_lookup_args->kvtag); */ @@ -7315,10 +7318,12 @@ PDC_Client_query_kvtag_server(uint32_t server_id, const pdc_kvtag_t *kvtag, int if (kvtag->value == NULL) { in.value = " "; + in.type = PDC_STRING; in.size = 1; } else { in.value = kvtag->value; + in.type = kvtag->type; in.size = kvtag->size; } @@ -7576,7 +7581,8 @@ PDCcont_get_objids(pdcid_t cont_id ATTRIBUTE(unused), int *nobj ATTRIBUTE(unused } perr_t -PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_size) +PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, + psize_t value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t kvtag; @@ -7585,6 +7591,7 @@ PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_ kvtag.name = tag_name; kvtag.value = (void *)tag_value; + kvtag.type = value_type; kvtag.size = (uint64_t)value_size; ret_value = PDC_add_kvtag(cont_id, &kvtag, 1); @@ -7598,7 +7605,8 @@ PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_ } perr_t -PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, psize_t *value_size) +PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t *kvtag = NULL; @@ -7610,6 +7618,7 @@ PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, psize_t *valu PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Error with PDC_get_kvtag", pdc_client_mpi_rank_g); *tag_value = kvtag->value; + *value_type = kvtag->type; *value_size = kvtag->size; done: @@ -7772,7 +7781,7 @@ PDC_Client_del_metadata(pdcid_t obj_id, int is_cont) } perr_t -PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_size) +PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, psize_t value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t kvtag; @@ -7781,6 +7790,7 @@ PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_si kvtag.name = tag_name; kvtag.value = (void *)tag_value; + kvtag.type = value_type; kvtag.size = (uint64_t)value_size; ret_value = PDC_add_kvtag(obj_id, &kvtag, 0); @@ -7793,7 +7803,8 @@ PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_si } perr_t -PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, psize_t *value_size) +PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size) { perr_t ret_value = SUCCEED; pdc_kvtag_t *kvtag = NULL; @@ -7805,6 +7816,7 @@ PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, psize_t *value_ PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Error with PDC_get_kvtag", pdc_client_mpi_rank_g); *tag_value = kvtag->value; + *value_type = kvtag->type; *value_size = kvtag->size; done: diff --git a/src/api/pdc_obj/include/pdc_cont.h b/src/api/pdc_obj/include/pdc_cont.h index 844b15425..33e924c14 100644 --- a/src/api/pdc_obj/include/pdc_cont.h +++ b/src/api/pdc_obj/include/pdc_cont.h @@ -191,7 +191,8 @@ perr_t PDCcont_del(pdcid_t cont_id); * * \return Non-negative on success/Negative on failure */ -perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t value_size); +perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, + psize_t value_size); /** * *********** @@ -203,7 +204,8 @@ perr_t PDCcont_put_tag(pdcid_t cont_id, char *tag_name, void *tag_value, psize_t * * \return Non-negative on success/Negative on failure */ -perr_t PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, psize_t *value_size); +perr_t PDCcont_get_tag(pdcid_t cont_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size); /** * Deleta a tag from a container diff --git a/src/api/pdc_obj/include/pdc_obj.h b/src/api/pdc_obj/include/pdc_obj.h index f678adf7f..8ad7a285a 100644 --- a/src/api/pdc_obj/include/pdc_obj.h +++ b/src/api/pdc_obj/include/pdc_obj.h @@ -409,7 +409,8 @@ perr_t PDCobj_del(pdcid_t obj_id); * * \return Non-negative on success/Negative on failure */ -perr_t PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t value_size); +perr_t PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, pdc_var_type_t value_type, + psize_t value_size); /** * Get tag information @@ -421,7 +422,8 @@ perr_t PDCobj_put_tag(pdcid_t obj_id, char *tag_name, void *tag_value, psize_t v * * \return Non-negative on success/Negative on failure */ -perr_t PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, psize_t *value_size); +perr_t PDCobj_get_tag(pdcid_t obj_id, char *tag_name, void **tag_value, pdc_var_type_t *value_type, + psize_t *value_size); /** * Delete a tag from the object diff --git a/src/api/pdc_obj/include/pdc_prop_pkg.h b/src/api/pdc_obj/include/pdc_prop_pkg.h index db73120ac..52d80efa1 100644 --- a/src/api/pdc_obj/include/pdc_prop_pkg.h +++ b/src/api/pdc_obj/include/pdc_prop_pkg.h @@ -37,9 +37,10 @@ struct _pdc_cont_prop { }; typedef struct pdc_kvtag_t { - char * name; - uint32_t size; - void * value; + char * name; + uint32_t size; + pdc_var_type_t type; + void * value; } pdc_kvtag_t; struct _pdc_transform_state { diff --git a/src/api/pdc_obj/pdc_dt_conv.c b/src/api/pdc_obj/pdc_dt_conv.c index 0806919ee..f18e4aa43 100644 --- a/src/api/pdc_obj/pdc_dt_conv.c +++ b/src/api/pdc_obj/pdc_dt_conv.c @@ -34,10 +34,10 @@ PDC_UNKNOWN = -1, PDC_INT = 0, PDC_FLOAT = 1, PDC_DOUBLE = 2, -PDC_STRING = 3, -PDC_COMPOUND = 4, -PDC_ENUM = 5, -PDC_ARRAY = 6, +PDC_CHAR = 3, +PDC_STRING = 4, +PDC_BOOLEAN = 5, +PDC_SHORT = 6, */ /* Called if overflow is possible */ diff --git a/src/api/pdc_obj/pdc_obj.c b/src/api/pdc_obj/pdc_obj.c index 073ece24c..d402782f9 100644 --- a/src/api/pdc_obj/pdc_obj.c +++ b/src/api/pdc_obj/pdc_obj.c @@ -27,6 +27,7 @@ #include "pdc_malloc.h" #include "pdc_id_pkg.h" #include "pdc_cont.h" +#include "pdc_cont_pkg.h" #include "pdc_prop_pkg.h" #include "pdc_obj_pkg.h" #include "pdc_obj.h" diff --git a/src/commons/CMakeLists.txt b/src/commons/CMakeLists.txt new file mode 100644 index 000000000..9f8abf690 --- /dev/null +++ b/src/commons/CMakeLists.txt @@ -0,0 +1,210 @@ +#------------------------------------------------------------------------------ +# PDC Commons +#------------------------------------------------------------------------------ + +set(PDC_COMMON_LIBRARY_NAME pdc_commons CACHE INTERNAL "") + +#------------------------------------------------------------------------------ +# External dependencies +#------------------------------------------------------------------------------ + +# #set(PDC_EXT_LIB_DEPENDENCIES ${PDC_COMMON_LIBRARY_NAME} ${PDC_EXT_LIB_DEPENDENCIES}) +# set(PDC_EXT_INCLUDE_DEPENDENCIES ${CMAKE_CURRENT_SOURCE_DIR}/profiling) +# set(PDC_EXPORTED_LIBS pdcprof) + +if(THREADS_HAVE_PTHREAD_ARG) + set_property(TARGET ${PDC_COMMON_LIBRARY_NAME} PROPERTY COMPILE_OPTIONS "-pthread") + set_property(TARGET ${PDC_COMMON_LIBRARY_NAME} PROPERTY INTERFACE_COMPILE_OPTIONS "-pthread") +endif() + + +# Mercury +find_package(MERCURY REQUIRED) +if(MERCURY_FOUND) + message(STATUS "mercury dir = ${MERCURY_DIR}") + find_path(MERCURY_INCLUDE_DIR mercury.h HINTS ${MERCURY_DIR}) + find_library(MERCURY_LIBRARY mercury HINTS ${MERCURY_DIR}) + message(STATUS "mercury include dir = ${MERCURY_INCLUDE_DIR}") + message(STATUS "mercury lib = ${MERCURY_LIBRARY}") + set(PDC_EXT_INCLUDE_DEPENDENCIES ${MERCURY_INCLUDE_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + ) + set(PDC_EXT_LIB_DEPENDENCIES mercury ${PDC_EXT_LIB_DEPENDENCIES}) +endif() + +include_directories(${PDC_EXT_INCLUDE_DEPENDENCIES}) + +#------------------------------------------------------------------------------ +# Include directories +#------------------------------------------------------------------------------ + +# Get a list of all directories that contain header files +file(GLOB_RECURSE LOCAL_INCLUDE_DIRS "*.h") + +# Remove the /filename.h at the end of each directory +list(TRANSFORM LOCAL_INCLUDE_DIRS REPLACE "/[^/]*$" "") + +# Remove duplicates +list(REMOVE_DUPLICATES LOCAL_INCLUDE_DIRS) + +set(PDC_COMMONS_INCLUDE_DIRS + ${LOCAL_INCLUDE_DIRS} + ${PDC_INCLUDES_BUILD_TIME} + ${PROJECT_BINARY_DIR} + ${PDC_SOURCE_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} +) + +include_directories( + ${PDC_COMMONS_INCLUDE_DIRS} +) + +message(STATUS "PDC_COMMONS_INCLUDE_DIRS: ${PDC_COMMONS_INCLUDE_DIRS}") + +install( + FILES + ${CMAKE_BINARY_DIR}/pdc_config.h + DESTINATION + ${PDC_INSTALL_INCLUDE_DIR} + COMPONENT + headers +) + +#------------------------------------------------------------------------------ +# Configure module header files +#------------------------------------------------------------------------------ +# Set unique vars used in the autogenerated config file (symbol import/export) +if(BUILD_SHARED_LIBS) + set(PDC_BUILD_SHARED_LIBS 1) + set(PDC_LIBTYPE SHARED) +else() + set(PDC_BUILD_SHARED_LIBS 0) + set(PDC_LIBTYPE STATIC) +endif() + +if(PDC_ENABLE_TIMING) + add_definitions(-DPDC_TIMING=1) +endif() + +#------------------------------------------------------------------------------ +# Set sources +#------------------------------------------------------------------------------ + +# Collect all source files +file(GLOB_RECURSE PDC_COMMONS_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.c) +file(GLOB_RECURSE PDC_COMMONS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.h) + + +#------------------------------------------------------------------------------ +# Libraries +#------------------------------------------------------------------------------ +# PDC COMMONS + + +add_library(${PDC_COMMON_LIBRARY_NAME} ${PDC_LIBTYPE} ${PDC_COMMONS_SOURCES} ${PDC_COMMONS_HEADERS}) + +target_include_directories(${PDC_COMMON_LIBRARY_NAME} + PUBLIC "$" + $ +) + +target_link_libraries(${PDC_COMMON_LIBRARY_NAME} INTERFACE + ${PDC_EXT_LIB_DEPENDENCIES} +) + +set(PDC_EXPORTED_LIBS ${PDC_COMMON_LIBRARY_NAME} ${PDC_EXPORTED_LIBS}) + +#----------------------------------------------------------------------------- +# Specify project header files to be installed +#----------------------------------------------------------------------------- + + +set(PDC_PUBLIC_HEADERS "") +set(PUBLIC_HEADER_DIR_LIST + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/generic/include + ${CMAKE_CURRENT_SOURCE_DIR}/profiling/include +) + +foreach(_header_dir ${PUBLIC_HEADER_DIR_LIST}) + file(GLOB_RECURSE _dir_headers ${_header_dir}/*.h) + list(APPEND PDC_PUBLIC_HEADERS ${_dir_headers}) +endforeach() + +set(PDC_COMMONS_HEADERS + ${PDC_PUBLIC_HEADERS} + ${PROJECT_BINARY_DIR}/pdc_config_sys.h + ${PROJECT_BINARY_DIR}/pdc_config.h + ) + +#----------------------------------------------------------------------------- +# Add file(s) to CMake Install +#----------------------------------------------------------------------------- +install( + FILES + ${PDC_COMMONS_HEADERS} + DESTINATION + ${PDC_INSTALL_INCLUDE_DIR} + COMPONENT + headers +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install +#----------------------------------------------------------------------------- + +install( + TARGETS + ${PDC_COMMON_LIBRARY_NAME} + EXPORT + ${PDC_EXPORTED_TARGETS} + LIBRARY DESTINATION ${PDC_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${PDC_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${PDC_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${PDC_EXPORTED_TARGETS} + DESTINATION + ${PDC_INSTALL_DATA_DIR}/cmake/pdc + FILE + ${PDC_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +if(NOT PDC_EXTERNALLY_CONFIGURED) +EXPORT ( + TARGETS + ${PDC_EXPORTED_LIBS} + FILE + ${PDC_EXPORTED_TARGETS}.cmake +) +endif() + +#------------------------------------------------------------------------------ +# Set variables for parent scope +#------------------------------------------------------------------------------ +# Used by config.cmake.build.in and Testing +set(PDC_INCLUDES_BUILD_TIME + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + ${PDC_COMMONS_INCLUDE_DIRS} + PARENT_SCOPE +) + +# Used by config.cmake.install.in +set(PDC_INCLUDES_INSTALL_TIME + ${PDC_COMMONS_INCLUDE_DIRS} + ${PDC_INSTALL_INCLUDE_DIR} + ${PDC_EXT_INCLUDE_DEPENDENCIES} + PARENT_SCOPE +) + +set(PDC_COMMONS_LIBRARIES ${PDC_COMMON_LIBRARY_NAME} PARENT_SCOPE) \ No newline at end of file diff --git a/src/commons/generic/include/pdc_generic.h b/src/commons/generic/include/pdc_generic.h new file mode 100644 index 000000000..47322b553 --- /dev/null +++ b/src/commons/generic/include/pdc_generic.h @@ -0,0 +1,219 @@ +#ifndef PDC_GENERIC_H +#define PDC_GENERIC_H + +#include +#include +#include +#include +#include + +#ifndef __cplusplus +#if __STDC_VERSION__ >= 199901L +/* C99 or later */ +#include +#else +/* Pre-C99 */ +typedef enum { false = 0, true = 1 } bool; +#endif +#endif + +typedef enum { + PDC_UNKNOWN = -1, /* error */ + PDC_INT = 0, /* integer types (identical to int32_t) */ + PDC_FLOAT = 1, /* floating-point types */ + PDC_DOUBLE = 2, /* double types */ + PDC_CHAR = 3, /* character types */ + PDC_STRING = 4, /* string types */ + PDC_BOOLEAN = 5, /* boolean types */ + PDC_SHORT = 6, /* short types */ + PDC_UINT = 7, /* unsigned integer types (identical to uint32_t) */ + PDC_INT64 = 8, /* 64-bit integer types */ + PDC_UINT64 = 9, /* 64-bit unsigned integer types */ + PDC_INT16 = 10, /* 16-bit integer types */ + PDC_INT8 = 11, /* 8-bit integer types */ + PDC_UINT8 = 12, /* 8-bit unsigned integer types */ + PDC_UINT16 = 13, /* 16-bit unsigned integer types */ + PDC_INT32 = 14, /* 32-bit integer types */ + PDC_UINT32 = 15, /* 32-bit unsigned integer types */ + PDC_LONG = 16, /* long types */ + PDC_VOID_PTR = 17, /* void pointer type */ + PDC_SIZE_T = 18, /* size_t type */ + PDC_TYPE_COUNT = 19 /* this is the number of var types and has to be the last */ +} pdc_c_var_type_t; + +typedef pdc_c_var_type_t PDC_CType; + +typedef enum { + PDC_CLS_SCALAR, + PDC_CLS_ARRAY, + PDC_CLS_ENUM, // not implemented, users can use PDC_CT_INT + PDC_CLS_STRUCT, // not implemented, users can use embedded key value pairs for the members in a struct + PDC_CLS_UNION, // not implemented, users can use embedded key value pairs for the only one member value + // in a union. + PDC_CLS_POINTER, // not implemented, users can use PDC_CT_INT64_T to store the pointer address, but + // won't work for distributed memory. + PDC_CLS_FUNCTION, // not implemented, users can use PDC_CT_INT64_T to store the function address, but + // won't work for distributed memory. + PDC_CLS_COUNT // just the count of the enum. +} pdc_c_var_class_t; + +typedef pdc_c_var_class_t PDC_CType_Class; + +// clang-format off +static const size_t DataTypeSizes[PDC_TYPE_COUNT] = { + sizeof(int), + sizeof(float), + sizeof(double), + sizeof(char), + sizeof(char *), + sizeof(bool), + sizeof(short), + sizeof(unsigned int), + sizeof(int64_t), + sizeof(uint64_t), + sizeof(int16_t), + sizeof(int8_t), + sizeof(uint8_t), + sizeof(uint16_t), + sizeof(int32_t), + sizeof(uint32_t), + sizeof(long), + sizeof(void *), + sizeof(size_t) +}; + +static const char *DataTypeNames[PDC_TYPE_COUNT] = { + "int", + "float", + "double", + "char", + "char*", + "bool", + "short", + "unsigned int", + "int64_t", + "uint64_t", + "int16_t", + "int8_t", + "uint8_t", + "uint16_t", + "int32_t", + "uint32_t", + "long", + "void*", + "size_t" +}; + +static const char *DataTypeEnumNames[PDC_TYPE_COUNT] = { + "PDC_INT", + "PDC_FLOAT", + "PDC_DOUBLE", + "PDC_CHAR", + "PDC_STRING", + "PDC_BOOLEAN", + "PDC_SHORT", + "PDC_UINT", + "PDC_INT64", + "PDC_UINT64", + "PDC_INT16", + "PDC_INT8", + "PDC_UINT8", + "PDC_UINT16", + "PDC_INT32", + "PDC_UINT32", + "PDC_LONG", + "PDC_VOID_PTR", + "PDC_SIZE_T" +}; + +static const char *DataTypeFormat[PDC_TYPE_COUNT] = { + "%d", // int + "%f", // float + "%lf", // double + "%c", // char + "%s", // char* + "%d", // bool (represented as an integer) + "%hd", // short + "%u", // unsigned int + "%lld", // int64_t + "%llu", // uint64_t + "%hd", // int16_t + "%hhd", // int8_t + "%hhu", // uint8_t + "%hu", // uint16_t + "%d", // int32_t + "%u", // uint32_t + "%ld", // long + "%p", // void* (pointer) + "%zu" // size_t +}; + +// clang-format on + +static const char * +get_enum_name_by_dtype(pdc_c_var_type_t type) +{ + if (type < 0 || type >= PDC_TYPE_COUNT) { + return NULL; + } + return DataTypeEnumNames[type]; +} + +static const size_t +get_size_by_dtype(pdc_c_var_type_t type) +{ + if (type < 0 || type >= PDC_TYPE_COUNT) { + return 0; + } + return DataTypeSizes[type]; +} + +static const size_t +get_size_by_class_n_type(void *data, size_t item_count, pdc_c_var_class_t pdc_class, + pdc_c_var_type_t pdc_type) +{ + size_t size = 0; + if (pdc_class == PDC_CLS_SCALAR) { + if (pdc_type == PDC_STRING) { + size = (strlen((char *)data) + 1) * sizeof(char); + } + else { + size = get_size_by_dtype(pdc_type); + } + } + else if (pdc_class == PDC_CLS_ARRAY) { + if (pdc_type == PDC_STRING) { + char **str_arr = (char **)data; + int i = 0; + for (i = 0; i < item_count; i++) { + size = size + (strlen(str_arr[i]) + 1) * sizeof(char); + } + } + else { + size = item_count * get_size_by_dtype(pdc_type); + } + } + return size; +} + +static const char * +get_name_by_dtype(pdc_c_var_type_t type) +{ + if (type < 0 || type >= PDC_TYPE_COUNT) { + return NULL; + } + return DataTypeNames[type]; +} + +static pdc_c_var_type_t +get_dtype_by_enum_name(const char *enumName) +{ + for (int i = 0; i < PDC_TYPE_COUNT; i++) { + if (strcmp(DataTypeEnumNames[i], enumName) == 0) { + return (pdc_c_var_type_t)i; + } + } + return PDC_UNKNOWN; // assuming PDC_UNKNOWN is the enum value for "unknown" +} + +#endif /* PDC_GENERIC_H */ \ No newline at end of file diff --git a/src/api/include/pdc_public.h b/src/commons/include/pdc_public.h similarity index 66% rename from src/api/include/pdc_public.h rename to src/commons/include/pdc_public.h index 8c47976f0..6c135f657 100644 --- a/src/api/include/pdc_public.h +++ b/src/commons/include/pdc_public.h @@ -28,6 +28,7 @@ #include #include #include +#include "pdc_generic.h" /*******************/ /* Public Typedefs */ @@ -41,24 +42,7 @@ typedef int PDC_int_t; typedef float PDC_float_t; typedef double PDC_double_t; -typedef enum { - PDC_UNKNOWN = -1, /* error */ - PDC_INT = 0, /* integer types */ - PDC_FLOAT = 1, /* floating-point types */ - PDC_DOUBLE = 2, /* double types */ - PDC_CHAR = 3, /* character types */ - PDC_COMPOUND = 4, /* compound types */ - PDC_ENUM = 5, /* enumeration types */ - PDC_ARRAY = 6, /* Array types */ - PDC_UINT = 7, /* unsigned integer types */ - PDC_INT64 = 8, /* 64-bit integer types */ - PDC_UINT64 = 9, /* 64-bit unsigned integer types */ - PDC_INT16 = 10, - PDC_INT8 = 11, - PDC_UINT8 = 12, - PDC_UINT16 = 13, - NCLASSES = 14 /* this must be last */ -} pdc_var_type_t; +typedef pdc_c_var_type_t pdc_var_type_t; typedef enum { PDC_PERSIST, PDC_TRANSIENT } pdc_lifetime_t; diff --git a/src/api/profiling/CMakeLists.txt b/src/commons/profiling/CMakeLists.txt similarity index 92% rename from src/api/profiling/CMakeLists.txt rename to src/commons/profiling/CMakeLists.txt index 1b2ee8254..05e2c90b9 100644 --- a/src/api/profiling/CMakeLists.txt +++ b/src/commons/profiling/CMakeLists.txt @@ -1,7 +1,8 @@ #------------------------------------------------------------------------------ # Include source and build directories #------------------------------------------------------------------------------ -include_directories( +set(PROFILING_INCLUDE_DIRS + ${PDC_COMMON_INCLUDE_DIRS} ${PDC_INCLUDES_BUILD_TIME} ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR} @@ -21,6 +22,10 @@ include_directories( ${MERCURY_INCLUDE_DIR} ${FASTBIT_INCLUDE_DIR} ) +message(STATUS "PDC_COMMON_INCLUDE_DIRS: ${PDC_COMMON_INCLUDE_DIRS}") +include_directories( + ${PROFILING_INCLUDE_DIRS} +) install( FILES @@ -65,6 +70,8 @@ set(PDC_PROF_SRCS # PDCPROF add_library(pdcprof ${PDC_PROF_SRCS}) pdc_set_lib_options(pdcprof "pdcprof" ${PDC_LIBTYPE}) +target_include_directories(pdcprof PUBLIC "$" + $) set(PDC_EXPORTED_LIBS pdcprof ${PDC_EXPORTED_LIBS}) diff --git a/src/api/profiling/include/pdc_hashtab.h b/src/commons/profiling/include/pdc_hashtab.h similarity index 100% rename from src/api/profiling/include/pdc_hashtab.h rename to src/commons/profiling/include/pdc_hashtab.h diff --git a/src/api/profiling/include/pdc_stack_ops.h b/src/commons/profiling/include/pdc_stack_ops.h similarity index 100% rename from src/api/profiling/include/pdc_stack_ops.h rename to src/commons/profiling/include/pdc_stack_ops.h diff --git a/src/api/profiling/pdc_hashtab.c b/src/commons/profiling/pdc_hashtab.c similarity index 100% rename from src/api/profiling/pdc_hashtab.c rename to src/commons/profiling/pdc_hashtab.c diff --git a/src/api/profiling/pdc_stack_ops.c b/src/commons/profiling/pdc_stack_ops.c similarity index 100% rename from src/api/profiling/pdc_stack_ops.c rename to src/commons/profiling/pdc_stack_ops.c diff --git a/src/commons/serde/include/pdc_serde.h b/src/commons/serde/include/pdc_serde.h new file mode 100644 index 000000000..8c5efae9e --- /dev/null +++ b/src/commons/serde/include/pdc_serde.h @@ -0,0 +1,156 @@ +#ifndef PDC_SERDE_H +#define PDC_SERDE_H + +#include +#include +#include +#include "pdc_generic.h" + +#define MAX_KEYS 10 +#define MAX_BUFFER_SIZE 1000 + +typedef struct { + PDC_CType pdc_type; /**< Data type of the key */ + size_t size; /**< Size of the key */ + void * key; /**< Pointer to the key data */ +} PDC_SERDE_Key; + +typedef struct { + PDC_CType_Class pdc_class; /**< Class of the value */ + PDC_CType pdc_type; /**< Data type of the value */ + size_t size; // size of the data. If a string, it is strlen(data) + 1; + // if an array, it is the number of elements; + // if a struct, it is the totalSize of the data chunk of the struct, etc. + void *data; /**< Pointer to the value data */ +} PDC_SERDE_Value; + +typedef struct { + PDC_SERDE_Key *keys; /**< Array of keys */ + size_t numKeys; /**< Number of keys */ + size_t totalSize; /**< Total size of the header */ +} PDC_SERDE_Header; + +typedef struct { + size_t numValues; /**< Number of values */ + PDC_SERDE_Value *values; /**< Array of values */ + size_t totalSize; /**< Total size of the data */ +} PDC_SERDE_Data; + +typedef struct { + PDC_SERDE_Header *header; /**< Pointer to the header */ + PDC_SERDE_Data * data; /**< Pointer to the data */ + size_t totalSize; /**< Total size of the serialized data */ +} PDC_SERDE_SerializedData; + +/** + * @brief Initialize a serialized data structure + * + * @param initial_field_count Number of initial fields to allocate space for + * + * @return Pointer to the initialized PDC_SERDE_SerializedData structure + */ +PDC_SERDE_SerializedData *pdc_serde_init(int initial_field_count); + +/** + * @brief Append a key-value pair to the serialized data structure + * + * @param data Pointer to the PDC_SERDE_SerializedData structure + * @param key Pointer to the PDC_SERDE_Key structure representing the key + * @param value Pointer to the PDC_SERDE_Value structure representing the value + */ +void pdc_serde_append_key_value(PDC_SERDE_SerializedData *data, PDC_SERDE_Key *key, PDC_SERDE_Value *value); + +/** + * @brief get the total size of PDC_SERDE_SerializedData structure instance + * + * @param data Pointer to the PDC_SERDE_SerializedData structure instance + * + * @return total size of the PDC_SERDE_SerializedData structure instance + */ +size_t get_total_size_for_serialized_data(PDC_SERDE_SerializedData *data); + +/** + * @brief Serialize the data in the serialized data structure and return the buffer + * + * @param data Pointer to the PDC_SERDE_SerializedData structure + * + * @return Pointer to the buffer containing the serialized data + */ +void *pdc_serde_serialize(PDC_SERDE_SerializedData *data); + +/** + * @brief Deserialize the buffer and return the deserialized data structure + * + * @param buffer Pointer to the buffer containing the serialized data + * + * @return Pointer to the deserialized PDC_SERDE_SerializedData structure + */ +PDC_SERDE_SerializedData *pdc_serde_deserialize(void *buffer); + +/** + * @brief Free the memory allocated for the serialized data structure + * + * @param data Pointer to the PDC_SERDE_SerializedData structure to be freed + */ +void pdc_serde_free(PDC_SERDE_SerializedData *data); + +/** + * @brief Print the contents of the serialized data structure + * + * @param data Pointer to the PDC_SERDE_SerializedData structure to be printed + */ +void pdc_serde_print(PDC_SERDE_SerializedData *data); + +/** + * @brief Create a PDC_SERDE_Key structure + * + * @param key Pointer to the key data + * @param pdc_type Data type of the key. For SERDE_Key, we only support PDC_CLS_SCALAR class. + * @param size Size of the key data + * + * @return Pointer to the created PDC_SERDE_Key structure + */ +static inline PDC_SERDE_Key * +PDC_SERDE_KEY(void *key, PDC_CType pdc_type, size_t size) +{ + PDC_SERDE_Key *pdc_key = (PDC_SERDE_Key *)malloc(sizeof(PDC_SERDE_Key)); + size_t key_size = (size_t)get_size_by_class_n_type(key, size, PDC_CLS_SCALAR, pdc_type); + pdc_key->key = malloc(key_size); + memcpy(pdc_key->key, key, key_size); + pdc_key->pdc_type = pdc_type; + pdc_key->size = key_size; + return pdc_key; +} + +/** + * @brief Create a PDC_SERDE_Value structure + * + * @param data Pointer to the value data + * @param pdc_type Data type of the value + * @param pdc_class Class of the value + * @param size Size of the value data + * + * @return Pointer to the created PDC_SERDE_Value structure + */ +static inline PDC_SERDE_Value * +PDC_SERDE_VALUE(void *data, PDC_CType pdc_type, PDC_CType_Class pdc_class, size_t size) +{ + PDC_SERDE_Value *pdc_value = (PDC_SERDE_Value *)malloc(sizeof(PDC_SERDE_Value)); + size_t value_size = 0; + if (pdc_class == PDC_CLS_STRUCT) { + // TODO: we need to check if data is a valid PDC_SERDE_SerializedData structure. + PDC_SERDE_SerializedData *struct_data = (PDC_SERDE_SerializedData *)data; + size = struct_data->totalSize; + } + else { + value_size = (size_t)get_size_by_class_n_type(data, size, pdc_class, pdc_type); + } + pdc_value->data = malloc(value_size); + memcpy(pdc_value->data, data, value_size); + pdc_value->pdc_class = pdc_class; + pdc_value->pdc_type = pdc_type; + pdc_value->size = value_size; + return pdc_value; +} + +#endif /* PDC_SERDE_H */ \ No newline at end of file diff --git a/src/commons/serde/pdc_serde.c b/src/commons/serde/pdc_serde.c new file mode 100644 index 000000000..fe9e9aa64 --- /dev/null +++ b/src/commons/serde/pdc_serde.c @@ -0,0 +1,346 @@ +#include "pdc_serde.h" + +PDC_SERDE_SerializedData * +pdc_serde_init(int initial_field_count) +{ + PDC_SERDE_SerializedData *data = malloc(sizeof(PDC_SERDE_SerializedData)); + data->header = malloc(sizeof(PDC_SERDE_Header)); + data->header->keys = malloc(sizeof(PDC_SERDE_Key) * initial_field_count); + data->header->numKeys = 0; + data->header->totalSize = 0; + data->data = malloc(sizeof(PDC_SERDE_Data)); + data->data->values = malloc(sizeof(PDC_SERDE_Value) * initial_field_count); + data->data->numValues = 0; + data->data->totalSize = 0; + return data; +} + +void +pdc_serde_append_key_value(PDC_SERDE_SerializedData *data, PDC_SERDE_Key *key, PDC_SERDE_Value *value) +{ + data->header->keys[data->header->numKeys] = *key; + data->header->numKeys++; + // append type, size, and key + data->header->totalSize += (sizeof(int) + sizeof(size_t) + key->size); + data->data->values[data->data->numValues] = *value; + data->data->numValues++; + // append class, type, size, and data + data->data->totalSize += (sizeof(int) + sizeof(int) + sizeof(size_t) + value->size); +} + +size_t +get_total_size_for_serialized_data(PDC_SERDE_SerializedData *data) +{ + if (data->totalSize <= 0) { + size_t total_size = data->header->totalSize + data->data->totalSize + sizeof(size_t) * 6; + data->totalSize = total_size; + } + return data->totalSize; +} + +// clang-format off +/** + * This function serializes the entire PDC_SERDE_SerializedData structure. + * + * The overview of the serialized binary data layout is: + * +---------------------+---------------------+----------------------+---------------------+----------------------+----------------------+----------------------+----------------------+ + * | Size of the Header | Size of the Data | Number of Keys | Header Region | Data Offset | Number of Values | Data Region | Data Offset | + * | (size_t) | (size_t) | (size_t) | | (size_t) | (size_t) | | (size_t) | + * +---------------------+---------------------+----------------------+---------------------+----------------------+----------------------+----------------------+----------------------+ + * + * The first 2 field is called meta-header, which provides metadata about size of the header region and the size of the data region. + * Note that the size of the header region doesn't include the 'Number of Keys' field. + * Also, the size of the data region doesn't include the 'Data Offset' field. + * + * Then the following is the header region with two keys: + * +-----------------------+-------------------------+-----------------------------+---------------------------+--------------------------+-----------------------------+---------------------------+ + * | Number of Keys | Key 1 Type | Key 1 Size | Key 1 Data | Key 2 Type | Key 2 Size | Key 2 Data | + * | (size_t) | (int8_t) | (size_t) | (Variable size depending | (int8_t) | (size_t) | (Variable size depending | + * | | | | on Key 1 Size) | | | on Key 2 Size) | + * +-----------------------+-------------------------+-----------------------------+---------------------------+--------------------------+-----------------------------+---------------------------+ + * + * Then, the following is a header offset validation point and the data region with the final offset validation point. + * + * |----------------------------------------------------------------------------------------------------------| + * | Data Offset (size_t) | Number of Value Entries (size_t) | Value 1 Class (int8_t) | Value 1 Type (int8_t) | + * |----------------------------------------------------------------------------------------------------------| + * | Value 1 Size (size_t)| Value 1 Data (Variable size depending on Value 1 Size) | Value 2 Class (int8_t) | + * |----------------------------------------------------------------------------------------------------------| + * | Value 2 Type (int8_t)| Value 2 Size (size_t) | Value 2 Data (Variable size depending on Value 2 Size) | + * |----------------------------------------------------------------------------------------------------------| + * | ...repeated for the number of value entries in the data... | + * |----------------------------------------------------------------------------------------------------------| + * | Final Data Offset (size_t) | + * |----------------------------------------------------------------------------------------------------------| + * + * Please refer to `get_size_by_class_n_type` function in pdc_generic.h for size calculation on scalar values and array values. + * + */ +// clang-format on +void * +pdc_serde_serialize(PDC_SERDE_SerializedData *data) +{ + // The buffer contains: + // the size of the header (size_t) + + // the size of the data (size_t) + + // the number of keys (size_t) + + // the header region + + // the data offset (size_t) + + // the number of value entries (size_t) + + // the data region + void *buffer = malloc(get_total_size_for_serialized_data(data)); + // serialize the meta header, which contains only the size of the header and the size of the data region. + memcpy(buffer, &data->header->totalSize, sizeof(size_t)); + memcpy(buffer + sizeof(size_t), &data->data->totalSize, sizeof(size_t)); + + // serialize the header + // start with the number of keys + memcpy(buffer + sizeof(size_t) * 2, &data->header->numKeys, sizeof(size_t)); + // then the keys + size_t offset = sizeof(size_t) * 3; + for (int i = 0; i < data->header->numKeys; i++) { + int8_t pdc_type = (int8_t)(data->header->keys[i].pdc_type); + memcpy(buffer + offset, &pdc_type, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(buffer + offset, &data->header->keys[i].size, sizeof(size_t)); + offset += sizeof(size_t); + memcpy(buffer + offset, data->header->keys[i].key, data->header->keys[i].size); + offset += data->header->keys[i].size; + } + + // serialize the data offset, this is for validation purpose to see if header region is corrupted. + memcpy(buffer + offset, &offset, sizeof(size_t)); + offset += sizeof(size_t); + + // serialize the data + // start with the number of value entries + memcpy(buffer + offset, &data->data->numValues, sizeof(size_t)); + offset += sizeof(size_t); + // then the values + for (int i = 0; i < data->data->numValues; i++) { + int8_t pdc_class = (int8_t)data->data->values[i].pdc_class; + int8_t pdc_type = (int8_t)data->data->values[i].pdc_type; + memcpy(buffer + offset, &pdc_class, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(buffer + offset, &pdc_type, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(buffer + offset, &data->data->values[i].size, sizeof(size_t)); + offset += sizeof(size_t); + memcpy(buffer + offset, data->data->values[i].data, data->data->values[i].size); + offset += data->data->values[i].size; + } + // serialize the data offset again, this is for validation purpose to see if data region is corrupted. + memcpy(buffer + offset, &offset, sizeof(size_t)); + offset += sizeof(size_t); + return buffer; +} + +PDC_SERDE_SerializedData * +pdc_serde_deserialize(void *buffer) +{ + size_t offset = 0; + // read the meta header + size_t headerSize; + size_t dataSize; + memcpy(&headerSize, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + memcpy(&dataSize, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + + // read the header + size_t numKeys; + memcpy(&numKeys, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + PDC_SERDE_Header *header = malloc(sizeof(PDC_SERDE_Header)); + header->keys = malloc(sizeof(PDC_SERDE_Key) * numKeys); + header->numKeys = numKeys; + header->totalSize = headerSize; + for (int i = 0; i < numKeys; i++) { + int8_t pdc_type; + size_t size; + memcpy(&pdc_type, buffer + offset, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(&size, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + void *key = malloc(size); + memcpy(key, buffer + offset, size); + offset += size; + header->keys[i].key = key; + header->keys[i].pdc_type = (PDC_CType)pdc_type; + header->keys[i].size = size; + } + + // read the data offset + size_t dataOffset; + memcpy(&dataOffset, buffer + offset, sizeof(size_t)); + // check the data offset + if (dataOffset != offset) { + printf("Error: data offset does not match the expected offset.\n"); + return NULL; + } + offset += sizeof(size_t); + + // read the data + size_t numValues; + memcpy(&numValues, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + PDC_SERDE_Data *data = malloc(sizeof(PDC_SERDE_Data)); + data->values = malloc(sizeof(PDC_SERDE_Value) * numValues); + data->numValues = numValues; + data->totalSize = dataSize; + for (int i = 0; i < numValues; i++) { + int8_t pdc_class; + int8_t pdc_type; + size_t size; + memcpy(&pdc_class, buffer + offset, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(&pdc_type, buffer + offset, sizeof(int8_t)); + offset += sizeof(int8_t); + memcpy(&size, buffer + offset, sizeof(size_t)); + offset += sizeof(size_t); + void *value = malloc(size); + memcpy(value, buffer + offset, size); + offset += size; + data->values[i].data = value; + data->values[i].pdc_class = (PDC_CType_Class)pdc_class; + data->values[i].pdc_type = (PDC_CType)pdc_type; + data->values[i].size = size; + } + // check the total size + memcpy(&dataOffset, buffer + offset, sizeof(size_t)); + // check the data offset + if (dataOffset != offset) { + printf("Error: data offset does not match the expected offset.\n"); + return NULL; + } + offset += sizeof(size_t); + if (offset != headerSize + sizeof(size_t) * 6 + dataSize) { + printf("Error: total size does not match the expected size.\n"); + return NULL; + } + // create the serialized data + PDC_SERDE_SerializedData *serializedData = malloc(sizeof(PDC_SERDE_SerializedData)); + serializedData->header = header; + serializedData->data = data; + serializedData->totalSize = headerSize + dataSize + sizeof(size_t) * 6; + + return serializedData; +} + +void +pdc_serde_free(PDC_SERDE_SerializedData *data) +{ + for (int i = 0; i < data->header->numKeys; i++) { + free(data->header->keys[i].key); + } + free(data->header->keys); + for (int i = 0; i < data->data->numValues; i++) { + free(data->data->values[i].data); + } + free(data->data->values); + free(data->header); + free(data->data); + free(data); +} + +void +pdc_serde_print(PDC_SERDE_SerializedData *data) +{ + printf("Header:\n"); + printf(" numKeys: %zu\n", data->header->numKeys); + printf(" totalSize: %zu\n", data->header->totalSize); + for (int i = 0; i < data->header->numKeys; i++) { + printf(" key %d:\n", i); + printf(" type: %d\n", data->header->keys[i].pdc_type); + printf(" size: %zu\n", data->header->keys[i].size); + printf(" key: %s\n", (char *)data->header->keys[i].key); + } + printf("Data:\n"); + printf(" numValues: %zu\n", data->data->numValues); + printf(" totalSize: %zu\n", data->data->totalSize); + for (int i = 0; i < data->data->numValues; i++) { + printf(" value %d:\n", i); + printf(" class: %d\n", data->data->values[i].pdc_class); + printf(" type: %d\n", data->data->values[i].pdc_type); + printf(" size: %zu\n", data->data->values[i].size); + printf(" data: "); + if (data->data->values[i].pdc_class == PDC_STRING) { + printf("%s\n", (char *)data->data->values[i].data); + } + else { + printf("\n"); + } + } +} + +int +test_serde_framework() +{ + // Initialize a serialized data structure + PDC_SERDE_SerializedData *data = pdc_serde_init(5); + + // Create and append key-value pairs for different data types + char * intKey_str = "int"; + int intVal = 42; + PDC_SERDE_Key * intKey = PDC_SERDE_KEY(intKey_str, PDC_STRING, sizeof(intKey_str)); + PDC_SERDE_Value *intValue = PDC_SERDE_VALUE(&intVal, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); + pdc_serde_append_key_value(data, intKey, intValue); + + char * doubleKey_str = "double"; + double doubleVal = 3.14159; + PDC_SERDE_Key * doubleKey = PDC_SERDE_KEY(doubleKey_str, PDC_STRING, sizeof(doubleKey_str)); + PDC_SERDE_Value *doubleValue = PDC_SERDE_VALUE(&doubleVal, PDC_DOUBLE, PDC_CLS_SCALAR, sizeof(double)); + pdc_serde_append_key_value(data, doubleKey, doubleValue); + + char * strKey_str = "string"; + char * strVal = "Hello, World!"; + PDC_SERDE_Key * strKey = PDC_SERDE_KEY(strKey_str, PDC_STRING, (strlen(strKey_str) + 1) * sizeof(char)); + PDC_SERDE_Value *strValue = + PDC_SERDE_VALUE(strVal, PDC_STRING, PDC_CLS_SCALAR, (strlen(strVal) + 1) * sizeof(char)); + pdc_serde_append_key_value(data, strKey, strValue); + + char * arrayKey_str = "array"; + int intArray[3] = {1, 2, 3}; + PDC_SERDE_Key * arrayKey = PDC_SERDE_KEY(arrayKey_str, PDC_STRING, sizeof(arrayKey_str)); + PDC_SERDE_Value *arrayValue = PDC_SERDE_VALUE(intArray, PDC_INT, PDC_CLS_ARRAY, sizeof(int) * 3); + pdc_serde_append_key_value(data, arrayKey, arrayValue); + + typedef struct { + int x; + int y; + } Point; + + char *pointKey = "point"; + Point pointVal = {10, 20}; + + PDC_SERDE_SerializedData *point_data = pdc_serde_init(2); + PDC_SERDE_Key * x_name = PDC_SERDE_KEY("x", PDC_STRING, sizeof(char *)); + PDC_SERDE_Value * x_value = PDC_SERDE_VALUE(&pointVal.x, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); + + PDC_SERDE_Key * y_name = PDC_SERDE_KEY("y", PDC_STRING, sizeof(char *)); + PDC_SERDE_Value *y_value = PDC_SERDE_VALUE(&pointVal.y, PDC_INT, PDC_CLS_SCALAR, sizeof(int)); + + pdc_serde_append_key_value(point_data, x_name, x_value); + pdc_serde_append_key_value(point_data, y_name, y_value); + void *point_buffer = pdc_serde_serialize(point_data); + + PDC_SERDE_Key * structKey = PDC_SERDE_KEY(pointKey, PDC_STRING, sizeof(pointKey)); + PDC_SERDE_Value *structValue = PDC_SERDE_VALUE(point_buffer, PDC_VOID_PTR, PDC_CLS_STRUCT, sizeof(Point)); + pdc_serde_append_key_value(data, structKey, structValue); + + // Serialize the data + void *buffer = pdc_serde_serialize(data); + + // Deserialize the buffer + PDC_SERDE_SerializedData *deserializedData = pdc_serde_deserialize(buffer); + + // Print the deserialized data + pdc_serde_print(deserializedData); + + // Free the memory + pdc_serde_free(data); + pdc_serde_free(deserializedData); + free(buffer); + + return 0; +} \ No newline at end of file diff --git a/src/utils/include/pdc_id_pkg.h b/src/commons/utils/include/pdc_id_pkg.h similarity index 100% rename from src/utils/include/pdc_id_pkg.h rename to src/commons/utils/include/pdc_id_pkg.h diff --git a/src/utils/include/pdc_linkedlist.h b/src/commons/utils/include/pdc_linkedlist.h similarity index 99% rename from src/utils/include/pdc_linkedlist.h rename to src/commons/utils/include/pdc_linkedlist.h index 6fa7ce08a..a9de691b0 100644 --- a/src/utils/include/pdc_linkedlist.h +++ b/src/commons/utils/include/pdc_linkedlist.h @@ -43,8 +43,8 @@ #ifndef PDC_LINKEDLIST_H #define PDC_LINKEDLIST_H -#include "pdc_cont_pkg.h" -#include "pdc_cont.h" +// #include "pdc_cont_pkg.h" +// #include "pdc_cont.h" #include "mercury_thread_mutex.h" #include diff --git a/src/utils/include/pdc_malloc.h b/src/commons/utils/include/pdc_malloc.h similarity index 100% rename from src/utils/include/pdc_malloc.h rename to src/commons/utils/include/pdc_malloc.h diff --git a/src/utils/include/pdc_private.h b/src/commons/utils/include/pdc_private.h similarity index 99% rename from src/utils/include/pdc_private.h rename to src/commons/utils/include/pdc_private.h index b6ca3bcfd..b0fa48f14 100644 --- a/src/utils/include/pdc_private.h +++ b/src/commons/utils/include/pdc_private.h @@ -146,7 +146,7 @@ extern pbool_t err_occurred; /* Include a basic profiling interface */ #ifdef ENABLE_PROFILING -#include "stack_ops.h" +#include "pdc_stack_ops.h" #define FUNC_ENTER(X) \ do { \ diff --git a/src/utils/include/pdc_timing.h b/src/commons/utils/include/pdc_timing.h similarity index 100% rename from src/utils/include/pdc_timing.h rename to src/commons/utils/include/pdc_timing.h diff --git a/src/utils/pdc_malloc.c b/src/commons/utils/pdc_malloc.c similarity index 100% rename from src/utils/pdc_malloc.c rename to src/commons/utils/pdc_malloc.c diff --git a/src/utils/pdc_timing.c b/src/commons/utils/pdc_timing.c similarity index 100% rename from src/utils/pdc_timing.c rename to src/commons/utils/pdc_timing.c diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 449b64f10..abe3917c6 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -48,7 +48,6 @@ add_executable(pdc_server.exe ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c ${PDC_SOURCE_DIR}/src/utils/pdc_region_utils.c - ${PDC_SOURCE_DIR}/src/utils/pdc_timing.c ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_analysis_common.c ${PDC_SOURCE_DIR}/src/api/pdc_transform/pdc_transforms_common.c ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_hist_pkg.c @@ -70,9 +69,9 @@ endif() if(PDC_ENABLE_FASTBIT) message(STATUS "Enabled fastbit") - target_link_libraries(pdc_server.exe mercury pdcprof -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} ${FASTBIT_LIBRARY}/libfastbit.so) + target_link_libraries(pdc_server.exe mercury ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} ${FASTBIT_LIBRARY}/libfastbit.so) else() - target_link_libraries(pdc_server.exe mercury pdcprof -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES}) + target_link_libraries(pdc_server.exe mercury ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES}) endif() diff --git a/src/server/include/pdc_client_server_common.h b/src/server/include/pdc_client_server_common.h index a57e2169f..e125af9a5 100644 --- a/src/server/include/pdc_client_server_common.h +++ b/src/server/include/pdc_client_server_common.h @@ -1190,6 +1190,11 @@ hg_proc_pdc_kvtag_t(hg_proc_t proc, void *data) // HG_LOG_ERROR("Proc error"); return ret; } + ret = hg_proc_int8_t(proc, &struct_data->type); + if (ret != HG_SUCCESS) { + // HG_LOG_ERROR("Proc error"); + return ret; + } if (struct_data->size) { switch (hg_proc_get_op(proc)) { case HG_DECODE: diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index a0a7845a4..859ce7bb8 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -3096,7 +3096,7 @@ HG_TEST_RPC_CB(region_release, handle) size2 = HG_Bulk_get_size(remote_bulk_handle); if (size != size2) { error = 1; - printf("==PDC_SERVER: local size %lu, remote %lu\n", size, size2); + printf("==PDC_SERVER: local size %llu, remote %llu\n", size, size2); PGOTO_ERROR(HG_OTHER_ERROR, "===PDC SERVER: HG_TEST_RPC_CB(region_release, " "handle) local and remote bulk size does not match"); } @@ -3269,7 +3269,7 @@ HG_TEST_RPC_CB(region_release, handle) size2 = HG_Bulk_get_size(remote_bulk_handle); if (size != size2) { error = 1; - printf("==PDC_SERVER: local size %lu, remote %lu\n", size, size2); + printf("==PDC_SERVER: local size %llu, remote %llu\n", size, size2); /* PGOTO_ERROR(HG_OTHER_ERROR, "===PDC SERVER: HG_TEST_RPC_CB(region_release, * handle) local and remote bulk size does not match"); */ } @@ -6846,10 +6846,10 @@ PDC_kvtag_dup(pdc_kvtag_t *from, pdc_kvtag_t **to) if (from == NULL || to == NULL) PGOTO_DONE(FAIL); - (*to) = (pdc_kvtag_t *)calloc(1, sizeof(pdc_kvtag_t)); - (*to)->name = (char *)malloc(strlen(from->name) + 1); - (*to)->size = from->size; - + (*to) = (pdc_kvtag_t *)calloc(1, sizeof(pdc_kvtag_t)); + (*to)->name = (char *)malloc(strlen(from->name) + 1); + (*to)->size = from->size; + (*to)->type = from->type; (*to)->value = (void *)malloc(from->size); memcpy((void *)(*to)->name, (void *)from->name, strlen(from->name) + 1); memcpy((void *)(*to)->value, (void *)from->value, from->size); diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index a3020628a..449b31c06 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1224,6 +1224,7 @@ PDC_Server_checkpoint() fwrite(&key_len, sizeof(int), 1, file); fwrite(kvlist_elt->kvtag->name, key_len, 1, file); fwrite(&kvlist_elt->kvtag->size, sizeof(uint32_t), 1, file); + fwrite(&kvlist_elt->kvtag->type, sizeof(int8_t), 1, file); fwrite(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, 1, file); } @@ -1403,7 +1404,8 @@ PDC_Server_restart(char *filename) } // init hash table - PDC_Server_init_hash_table(); + // FIXME: check if we need to init the hash table again. + // PDC_Server_init_hash_table(); if (fread(&n_cont, sizeof(int), 1, file) != 1) { printf("Read failed for n_count\n"); @@ -1492,6 +1494,9 @@ PDC_Server_restart(char *filename) if (fread(&kvtag_list->kvtag->size, sizeof(uint32_t), 1, file) != 1) { printf("Read failed for kvtag_list->kvtag->size\n"); } + if (fread(&kvtag_list->kvtag->type, sizeof(int8_t), 1, file) != 1) { + printf("Read failed for kvtag_list->kvtag->type\n"); + } kvtag_list->kvtag->value = malloc(kvtag_list->kvtag->size); if (fread(kvtag_list->kvtag->value, kvtag_list->kvtag->size, 1, file) != 1) { printf("Read failed for kvtag_list->kvtag->value\n"); diff --git a/src/server/pdc_server_region/pdc_server_region_request_handler.h b/src/server/pdc_server_region/pdc_server_region_request_handler.h index f3dc64cde..04ab48481 100644 --- a/src/server/pdc_server_region/pdc_server_region_request_handler.h +++ b/src/server/pdc_server_region/pdc_server_region_request_handler.h @@ -811,7 +811,7 @@ HG_TEST_RPC_CB(transfer_request, handle) ret_value = HG_Respond(handle, NULL, NULL, &out); if (in.access_type == PDC_WRITE) { ret_value = HG_Bulk_create(info->hg_class, 1, &(local_bulk_args->data_buf), - &(local_bulk_args->total_mem_size), HG_BULK_READWRITE, + (const hg_size_t *)&(local_bulk_args->total_mem_size), HG_BULK_READWRITE, &(local_bulk_args->bulk_handle)); if (ret_value != HG_SUCCESS) { printf("Error at HG_TEST_RPC_CB(transfer_request, handle): @ line %d \n", __LINE__); @@ -881,7 +881,7 @@ HG_TEST_RPC_CB(transfer_request, handle) *((int *)(local_bulk_args->data_buf + sizeof(int)))); */ ret_value = HG_Bulk_create(info->hg_class, 1, &(local_bulk_args->data_buf), - &(local_bulk_args->total_mem_size), HG_BULK_READWRITE, + (const hg_size_t *)&(local_bulk_args->total_mem_size), HG_BULK_READWRITE, &(local_bulk_args->bulk_handle)); if (ret_value != HG_SUCCESS) { printf("Error at HG_TEST_RPC_CB(transfer_request, handle): @ line %d \n", __LINE__); diff --git a/src/tests/cont_tags.c b/src/tests/cont_tags.c index 04132c332..caaf87b49 100644 --- a/src/tests/cont_tags.c +++ b/src/tests/cont_tags.c @@ -36,8 +36,9 @@ main(int argc, char **argv) int rank = 0, size = 1; - char tag_value[128], tag_value2[128], *tag_value_ret; - psize_t value_size; + char tag_value[128], tag_value2[128], *tag_value_ret; + pdc_var_type_t value_type; + psize_t value_size; strcpy(tag_value, "some tag value"); strcpy(tag_value2, "some tag value 2 is longer than tag 1"); @@ -78,31 +79,31 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCcont_put_tag(cont, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCcont_put_tag(cont, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 1\n"); ret_value = 1; } - ret = PDCcont_put_tag(cont, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCcont_put_tag(cont, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 1\n"); ret_value = 1; } - ret = PDCcont_put_tag(cont2, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCcont_put_tag(cont2, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 2\n"); ret_value = 1; } - ret = PDCcont_put_tag(cont2, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCcont_put_tag(cont2, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 2\n"); ret_value = 1; } - ret = PDCcont_get_tag(cont, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 1\n"); ret_value = 1; @@ -112,7 +113,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCcont_get_tag(cont, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 1\n"); ret_value = 1; @@ -123,7 +124,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCcont_get_tag(cont2, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont2, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 2\n"); ret_value = 1; @@ -134,7 +135,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCcont_get_tag(cont2, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCcont_get_tag(cont2, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 2\n"); ret_value = 1; diff --git a/src/tests/kvtag_add_get.c b/src/tests/kvtag_add_get.c index 97eadffed..91686b9be 100644 --- a/src/tests/kvtag_add_get.c +++ b/src/tests/kvtag_add_get.c @@ -33,13 +33,14 @@ int main() { - pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; - pdc_kvtag_t kvtag1, kvtag2, kvtag3; - char * v1 = "value1"; - int v2 = 2; - double v3 = 3.45; - void * value1, *value2, *value3; - psize_t value_size; + pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; + pdc_kvtag_t kvtag1, kvtag2, kvtag3; + char * v1 = "value1"; + int v2 = 2; + double v3 = 3.45; + pdc_var_type_t type1, type2, type3; + void * value1, *value2, *value3; + psize_t value_size; // create a pdc pdc = PDCinit("pdc"); @@ -88,42 +89,45 @@ main() kvtag1.name = "key1string"; kvtag1.value = (void *)v1; + kvtag1.type = PDC_STRING; kvtag1.size = strlen(v1) + 1; kvtag2.name = "key2int"; kvtag2.value = (void *)&v2; + kvtag1.type = PDC_INT; kvtag2.size = sizeof(int); kvtag3.name = "key3double"; kvtag3.value = (void *)&v3; + kvtag1.type = PDC_DOUBLE; kvtag3.size = sizeof(double); - if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.size) < 0) + if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.type, kvtag1.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.size) < 0) + if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.type, kvtag2.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.size) < 0) + if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.type, kvtag3.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", kvtag1.name, (char *)value1); - if (PDCobj_get_tag(obj2, kvtag2.name, (void *)&value2, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, kvtag2.name, (void *)&value2, (void *)&type2, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%d] from o2\n", kvtag2.name, *(int *)value2); - if (PDCobj_get_tag(obj2, kvtag3.name, (void *)&value3, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, kvtag3.name, (void *)&value3, (void *)&type3, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%f] from o2\n", kvtag3.name, *(double *)value3); @@ -135,15 +139,16 @@ main() v1 = "New Value After Delete"; kvtag1.value = (void *)v1; + kvtag1.type = PDC_STRING; kvtag1.size = strlen(v1) + 1; - if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.size) < 0) + if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.type, kvtag1.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); /* PDC_free_kvtag(&value1); */ - if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, kvtag1.name, (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", kvtag1.name, (char *)value1); diff --git a/src/tests/kvtag_add_get_benchmark.c b/src/tests/kvtag_add_get_benchmark.c index ee5efdcbf..a682e30ee 100644 --- a/src/tests/kvtag_add_get_benchmark.c +++ b/src/tests/kvtag_add_get_benchmark.c @@ -220,7 +220,8 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value v = i + my_obj_s; for (j = 0; j < n_attr; j++) { sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", v, j); - if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], tag_value_len + 1) < 0) + if (PDCobj_put_tag(obj_ids[i], tag_name, (void *)tag_values[j], PDC_STRING, tag_value_len + 1) < + 0) printf("fail to add a kvtag to o%" PRIu64 "\n", v); } } @@ -238,12 +239,13 @@ add_n_tags(uint64_t my_obj, uint64_t my_obj_s, uint64_t n_attr, char **tag_value void get_object_tags(pdcid_t obj_id, uint64_t obj_name_v, uint64_t n_attr, void **tag_values, uint64_t *value_size) { - uint64_t i; - char tag_name[256]; + uint64_t i; + char tag_name[256]; + pdc_var_type_t tag_type; for (i = 0; i < n_attr; i++) { sprintf(tag_name, "tag%" PRIu64 ".%" PRIu64 "", obj_name_v, i); - if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], (void *)&value_size[i]) < 0) + if (PDCobj_get_tag(obj_id, tag_name, (void **)&tag_values[i], &tag_type, (void *)&value_size[i]) < 0) printf("fail to get a kvtag from o%" PRIu64 "\n", obj_name_v); } } diff --git a/src/tests/kvtag_add_get_scale.c b/src/tests/kvtag_add_get_scale.c index 280eda25b..dc14c597e 100644 --- a/src/tests/kvtag_add_get_scale.c +++ b/src/tests/kvtag_add_get_scale.c @@ -76,7 +76,8 @@ main(int argc, char *argv[]) double stime, total_time, percent_time; pdc_kvtag_t kvtag; void ** values; - size_t value_size; + pdc_var_type_t value_type; + size_t value_size; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); @@ -167,6 +168,7 @@ main(int argc, char *argv[]) // Add tags kvtag.name = "Group"; kvtag.value = (void *)&v; + kvtag.type = PDC_INT; kvtag.size = sizeof(int); #ifdef ENABLE_MPI @@ -175,7 +177,7 @@ main(int argc, char *argv[]) #endif for (i = 0; i < my_add_tag; i++) { v = i + my_add_tag_s; - if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) printf("fail to add a kvtag to o%d\n", i + my_obj_s); if (i % tag_1percent == 0) { @@ -208,7 +210,8 @@ main(int argc, char *argv[]) stime = MPI_Wtime(); #endif for (i = 0; i < my_query; i++) { - if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_size) < 0) + if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, + (void *)&value_size) < 0) printf("fail to get a kvtag from o%d\n", i + my_query_s); if (i % query_1percent == 0) { diff --git a/src/tests/kvtag_get.c b/src/tests/kvtag_get.c index a8fcf70f1..7ca1e7c0e 100644 --- a/src/tests/kvtag_get.c +++ b/src/tests/kvtag_get.c @@ -33,9 +33,10 @@ int main() { - pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; - pdc_kvtag_t *value1, *value2, *value3; - psize_t value_size; + pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; + pdc_kvtag_t * value1, *value2, *value3; + pdc_var_type_t type1, type2, type3; + psize_t value_size; // create a pdc pdc = PDCinit("pdc"); @@ -82,17 +83,17 @@ main() else printf("Fail to create object @ line %d!\n", __LINE__); - if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", value1->name, (char *)value1->value); - if (PDCobj_get_tag(obj2, "key2int", (void *)&value2, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, "key2int", (void *)&value2, (void *)&type2, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%d] from o2\n", value2->name, *(int *)value2->value); - if (PDCobj_get_tag(obj2, "key3double", (void *)&value3, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj2, "key3double", (void *)&value3, (void *)&type3, (void *)&value_size) < 0) printf("fail to get a kvtag from o2\n"); else printf("successfully retrieved a kvtag [%s] = [%f] from o2\n", value3->name, @@ -100,7 +101,7 @@ main() PDC_free_kvtag(&value1); - if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&value_size) < 0) + if (PDCobj_get_tag(obj1, "key1string", (void *)&value1, (void *)&type1, (void *)&value_size) < 0) printf("fail to get a kvtag from o1\n"); else printf("successfully retrieved a kvtag [%s] = [%s] from o1\n", value1->name, (char *)value1->value); diff --git a/src/tests/kvtag_query.c b/src/tests/kvtag_query.c index 88e6621d7..cf1e80dcb 100644 --- a/src/tests/kvtag_query.c +++ b/src/tests/kvtag_query.c @@ -91,32 +91,35 @@ main() kvtag1.name = "key1string"; kvtag1.value = (void *)v1; + kvtag1.type = PDC_STRING; kvtag1.size = strlen(v1) + 1; kvtag2.name = "key2int"; kvtag2.value = (void *)&v2; + kvtag2.type = PDC_INT; kvtag2.size = sizeof(int); kvtag3.name = "key3double"; kvtag3.value = (void *)&v3; + kvtag3.type = PDC_DOUBLE; kvtag3.size = sizeof(double); - if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.size) < 0) + if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.type, kvtag1.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj1, kvtag2.name, kvtag2.value, kvtag2.size) < 0) + if (PDCobj_put_tag(obj1, kvtag2.name, kvtag2.value, kvtag2.type, kvtag2.size) < 0) printf("fail to add a kvtag to o1\n"); else printf("successfully added a kvtag to o1\n"); - if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.size) < 0) + if (PDCobj_put_tag(obj2, kvtag2.name, kvtag2.value, kvtag2.type, kvtag2.size) < 0) printf("fail to add a kvtag to o2\n"); else printf("successfully added a kvtag to o2\n"); - if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.size) < 0) + if (PDCobj_put_tag(obj2, kvtag3.name, kvtag3.value, kvtag3.type, kvtag3.size) < 0) printf("fail to add a kvtag to o2\n"); else printf("successfully added a kvtag to o2\n"); diff --git a/src/tests/kvtag_query_scale.c b/src/tests/kvtag_query_scale.c index a42cd5a5a..b9e4a6ffa 100644 --- a/src/tests/kvtag_query_scale.c +++ b/src/tests/kvtag_query_scale.c @@ -129,6 +129,7 @@ main(int argc, char *argv[]) // Add tags kvtag.name = "Group"; kvtag.value = (void *)&v; + kvtag.type = PDC_INT; kvtag.size = sizeof(int); for (iter = 0; iter < round; iter++) { @@ -136,7 +137,7 @@ main(int argc, char *argv[]) v = iter; for (i = 0; i < my_add_tag; i++) { - if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.size) < 0) + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) printf("fail to add a kvtag to o%d\n", i + my_obj_s); } @@ -153,6 +154,7 @@ main(int argc, char *argv[]) kvtag.name = "Group"; kvtag.value = (void *)&v; + kvtag.type = PDC_INT; kvtag.size = sizeof(int); for (iter = 0; iter < round; iter++) { diff --git a/src/tests/obj_tags.c b/src/tests/obj_tags.c index 2651cf64e..5906e3dd9 100644 --- a/src/tests/obj_tags.c +++ b/src/tests/obj_tags.c @@ -42,9 +42,10 @@ main(int argc, char **argv) dims[0] = 64; dims[1] = 3; dims[2] = 4; - char tag_value[128], tag_value2[128], *tag_value_ret; - char cont_name[128], obj_name1[128], obj_name2[128]; - psize_t value_size; + char tag_value[128], tag_value2[128], *tag_value_ret; + char cont_name[128], obj_name1[128], obj_name2[128]; + pdc_var_type_t value_type; + psize_t value_size; strcpy(tag_value, "some tag value"); strcpy(tag_value2, "some tag value 2 is longer"); @@ -119,30 +120,30 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_put_tag(obj1, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCobj_put_tag(obj1, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 1\n"); ret_value = 1; } - ret = PDCobj_put_tag(obj1, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCobj_put_tag(obj1, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 1\n"); ret_value = 1; } - ret = PDCobj_put_tag(obj2, "some tag", tag_value, strlen(tag_value) + 1); + ret = PDCobj_put_tag(obj2, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 2\n"); ret_value = 1; } - ret = PDCobj_put_tag(obj2, "some tag 2", tag_value2, strlen(tag_value2) + 1); + ret = PDCobj_put_tag(obj2, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at object 2\n"); ret_value = 1; } - ret = PDCobj_get_tag(obj1, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj1, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 1\n"); ret_value = 1; @@ -153,7 +154,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_get_tag(obj1, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj1, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 1\n"); ret_value = 1; @@ -164,7 +165,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_get_tag(obj2, "some tag", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj2, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 2\n"); ret_value = 1; @@ -175,7 +176,7 @@ main(int argc, char **argv) ret_value = 1; } - ret = PDCobj_get_tag(obj2, "some tag 2", (void **)&tag_value_ret, &value_size); + ret = PDCobj_get_tag(obj2, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at object 2\n"); ret_value = 1; diff --git a/src/tests/pdc_transforms_lib.c b/src/tests/pdc_transforms_lib.c index f1a83f742..6397bdc70 100644 --- a/src/tests/pdc_transforms_lib.c +++ b/src/tests/pdc_transforms_lib.c @@ -6,21 +6,26 @@ * >> pdc_public.h * * typedef enum { - * PDC_UNKNOWN = -1, - * PDC_INT = 0, - * PDC_FLOAT = 1, - * PDC_DOUBLE = 2, - * PDC_STRING = 3, - * PDC_COMPOUND = 4, - * PDC_ENUM = 5, - * PDC_ARRAY = 6, - * PDC_UINT = 7, - * PDC_INT64 = 8, - * PDC_UINT64 = 9, - * PDC_INT16 = 10, - * PDC_INT8 = 11, - * NCLASSES = 12 - * } PDC_var_type_t; + * PDC_UNKNOWN = -1, * error * + * PDC_INT = 0, * integer types (identical to int32_t) * + * PDC_FLOAT = 1, * floating-point types * + * PDC_DOUBLE = 2, * double types * + * PDC_CHAR = 3, * character types * + * PDC_STRING = 4, * string types * + * PDC_BOOLEAN = 5, * boolean types * + * PDC_SHORT = 6, * short types * + * PDC_UINT = 7, * unsigned integer types (identical to uint32_t) * + * PDC_INT64 = 8, * 64-bit integer types * + * PDC_UINT64 = 9, * 64-bit unsigned integer types * + * PDC_INT16 = 10, * 16-bit integer types * + * PDC_INT8 = 11, * 8-bit integer types * + * PDC_UINT8 = 12, * 8-bit unsigned integer types * + * PDC_UINT16 = 13, * 16-bit unsigned integer types * + * PDC_LONG = 14, * long types * + * PDC_VOID_PTR = 15, * void pointer type * + * PDC_SIZE_T = 16, * size_t type * + * TYPE_COUNT = 17 * this is the number of var types and has to be the last * + * } pdc_c_var_type_t; */ static int diff --git a/src/tests/read_obj_shared.c b/src/tests/read_obj_shared.c index 4b0058dbd..6f56965d8 100644 --- a/src/tests/read_obj_shared.c +++ b/src/tests/read_obj_shared.c @@ -163,7 +163,7 @@ main(int argc, char **argv) offset[0] = rank * my_data_size; local_offset[0] = 0; mysize[0] = my_data_size; - printf("rank %d offset = %lu, length = %lu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); + printf("rank %d offset = %llu, length = %llu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); local_region = PDCregion_create(ndim, local_offset, mysize); global_region = PDCregion_create(ndim, offset, mysize); @@ -220,7 +220,7 @@ main(int argc, char **argv) offset[0] = rank * my_data_size; local_offset[0] = 0; mysize[0] = my_data_size; - printf("rank %d offset = %lu, length = %lu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); + printf("rank %d offset = %llu, length = %llu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); local_region = PDCregion_create(ndim, local_offset, mysize); global_region = PDCregion_create(ndim, offset, mysize); diff --git a/src/tests/vpicio_mts.c b/src/tests/vpicio_mts.c index eec66215b..65b7b4dc8 100644 --- a/src/tests/vpicio_mts.c +++ b/src/tests/vpicio_mts.c @@ -199,7 +199,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("\n#Step %d\n", i); + printf("\n#Step %llu\n", i); } #endif PDCprop_set_obj_time_step(obj_prop_xx, i); diff --git a/src/utils/pdc_interface.c b/src/utils/pdc_interface.c index 4cca6b0bc..218d6518e 100644 --- a/src/utils/pdc_interface.c +++ b/src/utils/pdc_interface.c @@ -25,6 +25,8 @@ #include "pdc_malloc.h" #include "pdc_id_pkg.h" #include "pdc_interface.h" +#include "pdc_cont_pkg.h" +#include "pdc_cont.h" #include #include diff --git a/tools/pdc_export.c b/tools/pdc_export.c index a67f444e3..9c0b572b2 100644 --- a/tools/pdc_export.c +++ b/tools/pdc_export.c @@ -7,6 +7,7 @@ #include #include #include "hdf5.h" +#include "pdc_generic.h" // #define ENABLE_MPI 1 @@ -240,45 +241,11 @@ get_data_type(int data_type) if (data_type == -1) { return "PDC_UNKNOWN"; } - else if (data_type == 0) { - return "PDC_INT"; - } - else if (data_type == 1) { - return "PDC_FLOAT"; - } - else if (data_type == 2) { - return "PDC_DOUBLE"; - } - else if (data_type == 3) { - return "PDC_CHAR"; - } - else if (data_type == 4) { - return "PDC_COMPOUND"; - } - else if (data_type == 5) { - return "PDC_ENUM"; - } - else if (data_type == 6) { - return "PDC_ARRAY"; - } - else if (data_type == 7) { - return "PDC_UINT"; - } - else if (data_type == 8) { - return "PDC_INT64"; - } - else if (data_type == 9) { - return "PDC_UINT64"; - } - else if (data_type == 10) { - return "PDC_INT16"; - } - else if (data_type == 11) { - return "PDC_INT16"; - } - else { + char *result = get_enum_name_by_dtype(data_type); + if (result == NULL) { return "NULL"; } + return result; } char * diff --git a/tools/pdc_import.c b/tools/pdc_import.c index f51f587c8..adf8cd46e 100644 --- a/tools/pdc_import.c +++ b/tools/pdc_import.c @@ -711,9 +711,10 @@ do_attr(hid_t aid, pdcid_t obj_id) char buf[MAX_NAME] = {0}; char read_buf[TAG_LEN_MAX] = {0}; // pdc_kvtag_t kvtag1; - char * tag_name; - void * tag_value; - size_t tag_size; + char * tag_name; + void * tag_value; + pdc_var_type_t value_type; + size_t tag_size; /* * Get the name of the attribute. @@ -735,7 +736,7 @@ do_attr(hid_t aid, pdcid_t obj_id) else { tag_size = H5Tget_size(atype); } - PDCobj_put_tag(obj_id, tag_name, tag_value, tag_size); + PDCobj_put_tag(obj_id, tag_name, tag_value, value_type, tag_size); /* * Get attribute information: dataspace, data type diff --git a/tools/pdc_ls.c b/tools/pdc_ls.c index 9d3db32fe..5963b33e2 100644 --- a/tools/pdc_ls.c +++ b/tools/pdc_ls.c @@ -201,45 +201,11 @@ get_data_type(int data_type) if (data_type == -1) { return "PDC_UNKNOWN"; } - else if (data_type == 0) { - return "PDC_INT"; - } - else if (data_type == 1) { - return "PDC_FLOAT"; - } - else if (data_type == 2) { - return "PDC_DOUBLE"; - } - else if (data_type == 3) { - return "PDC_CHAR"; - } - else if (data_type == 4) { - return "PDC_COMPOUND"; - } - else if (data_type == 5) { - return "PDC_ENUM"; - } - else if (data_type == 6) { - return "PDC_ARRAY"; - } - else if (data_type == 7) { - return "PDC_UINT"; - } - else if (data_type == 8) { - return "PDC_INT64"; - } - else if (data_type == 9) { - return "PDC_UINT64"; - } - else if (data_type == 10) { - return "PDC_INT16"; - } - else if (data_type == 11) { - return "PDC_INT16"; - } - else { + char *result = get_enum_name_by_dtype(data_type); + if (result == NULL) { return "NULL"; } + return result; } char * From ffaeb3e35ed2a6aa6b9ab31921b2f0fb19483f28 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Wed, 21 Jun 2023 19:33:30 -0400 Subject: [PATCH 196/216] LLSM Importer update: new job script + new data type update on kvtags (#92) * remove unnecessary install block from CMakeLists.txt * update output * Revert "update output" This reverts commit fe1f8b44995bc0dabd3b957e1032c2da26f56fdd. * build kvtag_add_get_scale * comment off free * update code * 1. kvtag_scale_add_get added \n 2. uint64_t support for obj/tag/query count \n 3. moving work assigning block downwards right before creating objects \n 4. everything is tested working * do while loop added, tested with 1m object and works * 1m objects test works, 10m object test fail as the original also fails * add new executable to test set * enlarge PDC_SERVER_ID_INTERVAL * update code * update console args * add p search test * add console arg for changing number of attributes per object * free allocated memory * fix query count issue * fix attr length definition * code refactored * code refactored * code refactored * code refactored * code refactored * code refactored * fix data type * fix data type * fix data type * add client side statistics * add client side statistics * fix format * clang formatter * update CMake * update CMake * update CMake * free allocated memory properly * clang format * clang format * clang-format-10 * change file name * address review comments * update llsm importer * update llsm importer * update server checkpoint intervals * update gitignore * adding job scripts * adding one debugging msg * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update output for uint64_t * add scripts * update output for uint64_t * update output for uint64_t * update output for uint64_t * update scripts * update scripts * delete debugging message * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * update tag names * update tag names * update query startingpos * update query startingpos * update job scripts * add progressive timing for kvtag_add_get_scale * fix iteration count in final report * update job scripts and benckmark program * update message format * update message format * update message format * update message format * clang format * update job scripts * comment off object/container close procedure in benchmark to save node hours * change the max number of object to 1M * change the max length of attribute value * change the max length of attribute value * llsm tiff import test * llsm tiff import test * llsm tiff import test * llsm tiff import test * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update cmake and llsm_importer * update cmake and llsm_importer * close if in cmake * cmake fix tiff * cmake policy to suppress warning * add pdc include dir * update code * update code * update code * update code * update code * update code * update array generating method * update array generating method * update array generating method * update array generating method * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * fix return type * fix return type * add timing * add timing * fix output * llsm tiff importer 1st version: read csv and import tiff files to PDC, adding metadata available in CSV files and TIFF loader * fix vairable name * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * add scripts * add scripts * add scripts * debugging for nonMPI program * debugging for nonMPI program * debugging for nonMPI program * clang format, without PDC, everything works perfectly. program fails at PDC init stage where PDCprop_create(PDC_CONT_CREATE, pdc) is being created * enable MPI * enable MPI * enlarge BCase size * enlarge BCase size * enlarge BCase size * resolve bcast count * llsm data path in script * llsm data path in script * update csv reader * update csv reader * update csv reader * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * enlarge max write * update pdc * update pdc * update pdc * update pdc * update pdc_import.c * update pdc_import.c * update pdc_export.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update tools/cmake * clang format * clang format * added a tutorial for llsm_importer * added a tutorial for llsm_importer * make sure the line feed is included for string attribute * update timing for overall completion time * update formatting * update metrics * forcibly enable openmp * adding C flags from the mex compiler * Update .gitlab-ci.yml * updated code * clang format * llsm_importer (#1) formatter on llsm_importer * add type for kvtag structure (#2) * upate metadata type system * update serde framework to coupe with the new data type system * replace unnecessary data types * adding type for pdc_kvtag_t, all occurances are fixed * update new commons CMake for publishing commons * commons compilation passed * compiled * Feature/metadata type (#3) * upate metadata type system * update serde framework to coupe with the new data type system * replace unnecessary data types * adding type for pdc_kvtag_t, all occurances are fixed * update new commons CMake for publishing commons * commons compilation passed * compiled * remove unnecessary header files from installation * resolve conflict * add important files * clang formatting * update cmake * update * print numWorkers * update scripts * update script * update script * formatting * update llsm_tools.c * remove unnecessary hash table init --------- Co-authored-by: Houjun Tang Co-authored-by: Jean Luca Bez --- scripts/llsm_importer/gen_script.sh | 24 ++++++++- scripts/llsm_importer/template.sh | 64 +++++++++++++++++++----- src/api/pdc_region/pdc_region_transfer.c | 3 +- src/server/pdc_server.c | 4 -- tools/llsm/parallelReadTiff.c | 8 +++ tools/llsm_importer.c | 22 ++++---- 6 files changed, 95 insertions(+), 30 deletions(-) diff --git a/scripts/llsm_importer/gen_script.sh b/scripts/llsm_importer/gen_script.sh index 9d310f2bb..efefd819f 100755 --- a/scripts/llsm_importer/gen_script.sh +++ b/scripts/llsm_importer/gen_script.sh @@ -1,5 +1,21 @@ #!/bin/bash -N_THREAD=NO + +# Per node configuration of your HPC system. +MAX_PYSICAL_CORE=128 +MAX_HYPERTHREADING=2 + +# Designated number of threads per process on each node +# (this should be associated with -c option in srun) +NUM_THREAD_PER_SERVER_PROC=128 +NUM_THREAD_PER_CLIENT_PROC=64 + + +# Designated number of processes for server anc client on each node +# (this should be associated with -n option in srun) +NUM_SERVER_PROC_PER_NODE=1 +NUM_CLIENT_PROC_PER_NODE=2 + + MAX_NODE=512 MAX_ATTR=1024 MAX_ATTRLEN=1000 @@ -13,6 +29,12 @@ for (( i = 1; i <= $MAX_NODE; i*=2 )); do cp template.sh $TARGET sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET sed -i "s/NODENUM/${i}/g" $TARGET + sed -i "s/MPHYSICALCORE/${MAX_PYSICAL_CORE}/g" $TARGET + sed -i "s/MHYPERTHREADING/${MAX_HYPERTHREADING}/g" $TARGET + sed -i "s/N_SERVER_PROC/${NUM_SERVER_PROC_PER_NODE}/g" $TARGET + sed -i "s/N_CLIENT_PROC/${NUM_CLIENT_PROC_PER_NODE}/g" $TARGET + sed -i "s/NTHREAD_PER_SPROC/${NUM_THREAD_PER_SERVER_PROC}/g" $TARGET + sed -i "s/NTHREAD_PER_CPROC/${NUM_THREAD_PER_CLIENT_PROC}/g" $TARGET if [[ "$i" -gt "16" ]]; then sed -i "s/REG//g" $TARGET else diff --git a/scripts/llsm_importer/template.sh b/scripts/llsm_importer/template.sh index d736b4076..834b8e2e5 100755 --- a/scripts/llsm_importer/template.sh +++ b/scripts/llsm_importer/template.sh @@ -13,24 +13,52 @@ # export PDC_DEBUG=0 - -export PDC_TMPDIR=$SCRATCH/data/pdc/conf - -rm -rf $PDC_TMPDIR/* - +# This is a script for running PDC in shared mode on Perlmutter +# When running in Shared mode, the client processes and server processes are running on the same node. +# By alternating the number of server processes and the number client processes, you should be able to change the C/S ratio. +# You can simply set the number of server processes, and let the script to calculate the number of client processes. + +# Per node configuration of your HPC system. +MAX_PYSICAL_CORE=MPHYSICALCORE +MAX_HYPERTHREADING=MHYPERTHREADING + +# Designated number of threads per process on each node +# (this should be associated with -c option in srun) +NUM_THREAD_PER_SERVER_PROC=NTHREAD_PER_SPROC +NUM_THREAD_PER_CLIENT_PROC=NTHREAD_PER_CPROC + + +# Designated number of processes for server anc client on each node +# (this should be associated with -n option in srun) +NUM_SERVER_PROC_PER_NODE=N_SERVER_PROC +NUM_CLIENT_PROC_PER_NODE=N_CLIENT_PROC + +# test if the number of threads is no larger than the total number of logical cores +TOTAL_NUM_PROC_PER_NODE=$((NUM_THREAD_PER_SERVER_PROC * NUM_SERVER_PROC_PER_NODE + NUM_THREAD_PER_CLIENT_PROC * NUM_CLIENT_PROC_PER_NODE)) +TOTAL_NUM_LOGICAL_CORE_PER_NODE=$((MAX_PYSICAL_CORE * MAX_HYPERTHREADING)) +if [[ "$TOTAL_NUM_PROC_PER_NODE" -gt "$TOTAL_NUM_LOGICAL_CORE_PER_NODE" ]]; then + echo "Error: TOTAL_NUM_PROC_PER_NODE is larger than TOTAL_NUM_LOGICAL_CORE_PER_NODE" + TOTAL_AVAILABLE_CORE=$((TOTAL_NUM_LOGICAL_CORE_PER_NODE - NUM_THREAD_PER_SERVER_PROC * NUM_SERVER_PROC_PER_NODE)) + NUM_CLIENT_PROC_PER_NODE=$(( TOTAL_AVAILABLE_CORE / NUM_THREAD_PER_CLIENT_PROC)) + echo "fixing the number of client processes to $NUM_CLIENT_PROC_PER_NODE" +fi + +# Set the number of times the test should be repeated. REPEAT=1 +# calculate the number of total processes for both server side and client side. N_NODE=NODENUM -NCLIENT=1 -# NCLIENT=126 +NCLIENT=$((NUM_CLIENT_PROC_PER_NODE * N_NODE)) +NSERVER=$((NUM_SERVER_PROC_PER_NODE * N_NODE)) +# clean up the PDC tmp directory +export PDC_TMPDIR=$SCRATCH/data/pdc/conf +rm -rf $PDC_TMPDIR/* export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE mkdir -p $PDC_TMPDIR -let TOTALPROC=$NCLIENT*$N_NODE - EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin -TOOLPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/source/pdc_llsm/tools/build +TOOLPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/source/pdc/tools/build SERVER=$EXECPATH/pdc_server.exe CLIENT=$TOOLPATH/llsm_importer CLOSE=$EXECPATH/close_server @@ -44,19 +72,31 @@ IMGLIST_PATH=${LLSM_DATA_PATH}/ImageList_from_encoder.csv date +# OpenMP settings: +# set the OPENMP thread number to the smaller number between $NUM_THREAD_PER_SERVER_PROC and $NUM_THREAD_PER_CLIENT_PROC +export OMP_NUM_THREADS=$((NUM_THREAD_PER_SERVER_PROC < NUM_THREAD_PER_CLIENT_PROC ? NUM_THREAD_PER_SERVER_PROC : NUM_THREAD_PER_CLIENT_PROC)) +export OMP_PLACES=threads +export OMP_PROC_BIND=close + +echo "OMP_NUM_THREADS=$OMP_NUM_THREADS" +echo "NSERVER=$NSERVER" +echo "NUM_THREAD_PER_SERVER_PROC=$NUM_THREAD_PER_SERVER_PROC" +echo "NCLIENT=$NCLIENT" +echo "NUM_THREAD_PER_CLIENT_PROC=$NUM_THREAD_PER_CLIENT_PROC" + echo "" echo "=============" echo "$i Init server" echo "=============" -stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $NSERVER -c $NUM_THREAD_PER_SERVER_PROC --cpu_bind=cores $SERVER & sleep 5 echo "============================================" echo "KVTAGS with $N_NODE nodes" echo "============================================" -stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT -f $IMGLIST_PATH +stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $NCLIENT -c $NUM_THREAD_PER_CLIENT_PROC --cpu_bind=cores $CLIENT -f $IMGLIST_PATH echo "" echo "=================" diff --git a/src/api/pdc_region/pdc_region_transfer.c b/src/api/pdc_region/pdc_region_transfer.c index 775947f5d..87f9f3625 100644 --- a/src/api/pdc_region/pdc_region_transfer.c +++ b/src/api/pdc_region/pdc_region_transfer.c @@ -1446,8 +1446,7 @@ PDCregion_transfer_start(pdcid_t transfer_request_id) // Pack local region to a contiguous memory buffer unit = transfer_request->unit; - // Convert user buf into a contiguous buffer called new_buf, which is determined by the shape of local - // objects. + // Convert user buf into a contiguous buffer called , which is determined by the shape of local objects. pack_region_buffer(transfer_request->buf, transfer_request->obj_dims, transfer_request->total_data_size, transfer_request->local_region_ndim, transfer_request->local_region_offset, transfer_request->local_region_size, unit, transfer_request->access_type, diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 449b31c06..e08882120 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1403,10 +1403,6 @@ PDC_Server_restart(char *filename) printf("Error getting slurm job id from SLURM_JOB_ID!\n"); } - // init hash table - // FIXME: check if we need to init the hash table again. - // PDC_Server_init_hash_table(); - if (fread(&n_cont, sizeof(int), 1, file) != 1) { printf("Read failed for n_count\n"); } diff --git a/tools/llsm/parallelReadTiff.c b/tools/llsm/parallelReadTiff.c index c81928584..cc4026ac9 100644 --- a/tools/llsm/parallelReadTiff.c +++ b/tools/llsm/parallelReadTiff.c @@ -31,6 +31,8 @@ readTiffParallelBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, vo int32_t batchSize = (z - 1) / numWorkers + 1; uint64_t bytes = bits / 8; + printf("numWorkers %d\n", numWorkers); + int32_t w; #ifdef ENABLE_OPENMP #pragma omp parallel for @@ -105,6 +107,8 @@ readTiffParallel(uint64_t x, uint64_t y, uint64_t z, const char *fileName, void int32_t batchSize = (z - 1) / numWorkers + 1; uint64_t bytes = bits / 8; + printf("numWorkers %d\n", numWorkers); + uint16_t compressed = 1; TIFF * tif = TIFFOpen(fileName, "r"); TIFFGetField(tif, TIFFTAG_COMPRESSION, &compressed); @@ -321,6 +325,8 @@ readTiffParallel2DBak(uint64_t x, uint64_t y, uint64_t z, const char *fileName, int32_t batchSize = (y - 1) / numWorkers + 1; uint64_t bytes = bits / 8; + printf("numWorkers %d\n", numWorkers); + int32_t w; #ifdef ENABLE_OPENMP #pragma omp parallel for @@ -403,6 +409,8 @@ readTiffParallel2D(uint64_t x, uint64_t y, uint64_t z, const char *fileName, voi uint8_t errBak = 0; char errString[10000]; + printf("numWorkers %d\n", numWorkers); + #ifdef ENABLE_OPENMP #pragma omp parallel for #endif diff --git a/tools/llsm_importer.c b/tools/llsm_importer.c index ff358fcd3..ea5097278 100644 --- a/tools/llsm_importer.c +++ b/tools/llsm_importer.c @@ -138,14 +138,14 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) switch (data_type) { case 'i': int ivalue = atoi(field_value); - PDCobj_put_tag(cur_obj_g, field_name, &ivalue, sizeof(int)); + PDCobj_put_tag(cur_obj_g, field_name, &ivalue, PDC_INT, sizeof(int)); break; case 'f': double fvalue = atof(field_value); - PDCobj_put_tag(cur_obj_g, field_name, &fvalue, sizeof(double)); + PDCobj_put_tag(cur_obj_g, field_name, &fvalue, PDC_DOUBLE, sizeof(double)); break; case 's': - PDCobj_put_tag(cur_obj_g, field_name, field_value, sizeof(char) * (strlen(field_value) + 1)); + PDCobj_put_tag(cur_obj_g, field_name, field_value, PDC_STRING, strlen(field_value)); break; default: break; @@ -154,14 +154,14 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) } // add extra metadata tags based on the image_info struct - PDCobj_put_tag(cur_obj_g, "x", &(image_info->x), sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "y", &(image_info->y), sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "z", &(image_info->z), sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "bits", &(image_info->bits), sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "startSlice", &(image_info->startSlice), sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "stripeSize", &(image_info->stripeSize), sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "is_imageJ", &(image_info->is_imageJ), sizeof(uint64_t)); - PDCobj_put_tag(cur_obj_g, "imageJ_Z", &(image_info->imageJ_Z), sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "x", &(image_info->x), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "y", &(image_info->y), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "z", &(image_info->z), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "bits", &(image_info->bits), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "startSlice", &(image_info->startSlice), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "stripeSize", &(image_info->stripeSize), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "is_imageJ", &(image_info->is_imageJ), PDC_UINT64, sizeof(uint64_t)); + PDCobj_put_tag(cur_obj_g, "imageJ_Z", &(image_info->imageJ_Z), PDC_UINT64, sizeof(uint64_t)); // close object PDCobj_close(cur_obj_g); From 39fcd7c2a6d50bd342973db58e1346cfb91f52d8 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Fri, 23 Jun 2023 09:09:38 -0700 Subject: [PATCH 197/216] Update .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39f940487..90d60224c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -164,7 +164,7 @@ perlmutter-metrics: variables: PDC_N_NODES: 64 PDC_N_CLIENTS: 127 - SCHEDULER_PARAMETERS: "-A m1248 --qos=debug --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00" + SCHEDULER_PARAMETERS: "-A m1248 --qos=regular --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00" SUPERCOMPUTER: "perlmutter" MERCURY_DIR: "/global/cfs/cdirs/m1248/pdc-perlmutter/mercury/install" PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-metrics" From 46abeed40fbbfc2cbabfbf94b8d9eac46a52d9e1 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Fri, 23 Jun 2023 17:27:19 -0400 Subject: [PATCH 198/216] fix warnings, commenting off 'find_path' and 'find_library' for Mercury in src/commons/CMakeLists.txt (#93) * Revert "update output" This reverts commit fe1f8b44995bc0dabd3b957e1032c2da26f56fdd. * build kvtag_add_get_scale * comment off free * update code * 1. kvtag_scale_add_get added \n 2. uint64_t support for obj/tag/query count \n 3. moving work assigning block downwards right before creating objects \n 4. everything is tested working * do while loop added, tested with 1m object and works * 1m objects test works, 10m object test fail as the original also fails * add new executable to test set * enlarge PDC_SERVER_ID_INTERVAL * update code * update console args * add p search test * add console arg for changing number of attributes per object * free allocated memory * fix query count issue * fix attr length definition * code refactored * code refactored * code refactored * code refactored * code refactored * code refactored * fix data type * fix data type * fix data type * add client side statistics * add client side statistics * fix format * clang formatter * update CMake * update CMake * update CMake * free allocated memory properly * clang format * clang format * clang-format-10 * change file name * address review comments * update llsm importer * update llsm importer * update server checkpoint intervals * update gitignore * adding job scripts * adding one debugging msg * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update container creation to collective mode for debugging purpose * update output for uint64_t * add scripts * update output for uint64_t * update output for uint64_t * update output for uint64_t * update scripts * update scripts * delete debugging message * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * make Cmake to publish scripts directory * update tag names * update tag names * update query startingpos * update query startingpos * update job scripts * add progressive timing for kvtag_add_get_scale * fix iteration count in final report * update job scripts and benckmark program * update message format * update message format * update message format * update message format * clang format * update job scripts * comment off object/container close procedure in benchmark to save node hours * change the max number of object to 1M * change the max length of attribute value * change the max length of attribute value * llsm tiff import test * llsm tiff import test * llsm tiff import test * llsm tiff import test * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update code * update cmake and llsm_importer * update cmake and llsm_importer * close if in cmake * cmake fix tiff * cmake policy to suppress warning * add pdc include dir * update code * update code * update code * update code * update code * update code * update array generating method * update array generating method * update array generating method * update array generating method * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * update CMakeLists * fix return type * fix return type * add timing * add timing * fix output * llsm tiff importer 1st version: read csv and import tiff files to PDC, adding metadata available in CSV files and TIFF loader * fix vairable name * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * fix cmake * add scripts * add scripts * add scripts * debugging for nonMPI program * debugging for nonMPI program * debugging for nonMPI program * clang format, without PDC, everything works perfectly. program fails at PDC init stage where PDCprop_create(PDC_CONT_CREATE, pdc) is being created * enable MPI * enable MPI * enlarge BCase size * enlarge BCase size * enlarge BCase size * resolve bcast count * llsm data path in script * llsm data path in script * update csv reader * update csv reader * update csv reader * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * update pdc * enlarge max write * update pdc * update pdc * update pdc * update pdc * update pdc_import.c * update pdc_import.c * update pdc_export.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update pdc_import.c * update tools/cmake * clang format * clang format * added a tutorial for llsm_importer * added a tutorial for llsm_importer * make sure the line feed is included for string attribute * update timing for overall completion time * update formatting * update metrics * forcibly enable openmp * adding C flags from the mex compiler * Update .gitlab-ci.yml * updated code * clang format * llsm_importer (#1) formatter on llsm_importer * add type for kvtag structure (#2) * upate metadata type system * update serde framework to coupe with the new data type system * replace unnecessary data types * adding type for pdc_kvtag_t, all occurances are fixed * update new commons CMake for publishing commons * commons compilation passed * compiled * Feature/metadata type (#3) * upate metadata type system * update serde framework to coupe with the new data type system * replace unnecessary data types * adding type for pdc_kvtag_t, all occurances are fixed * update new commons CMake for publishing commons * commons compilation passed * compiled * remove unnecessary header files from installation * resolve conflict * add important files * clang formatting * update cmake * update * print numWorkers * update scripts * update script * update script * formatting * update llsm_tools.c * remove unnecessary hash table init * update script * fix some warnings * fix some warnings * update * update * fix warning * update * update * update * update * update * update * update * update * update * fix warnings * fix warnings * fix warnings * fix warnings * fix warnings * fix warnings * fix warnings * update * update * update * server address and file paths using 1024, TMP_DIR path using 1024/2, NAME_MAX for appname and objname takes 1024/2, HOSTNAME takes 1024/8, NA_INFO_STRING takes 1024/4 * update * update * update * update --------- Co-authored-by: Houjun Tang Co-authored-by: Jean Luca Bez --- CMakeLists.txt | 40 +++++++++++++++++++ README.md | 5 ++- scripts/llsm_importer/gen_script.sh | 6 +-- src/api/CMakeLists.txt | 9 +++++ src/api/pdc_client_connect.c | 4 +- src/commons/CMakeLists.txt | 8 +++- src/commons/generic/include/pdc_generic.h | 36 ++++++++--------- src/commons/serde/include/pdc_serde.h | 27 +++++++------ src/commons/serde/pdc_serde.c | 30 +++++++------- src/server/CMakeLists.txt | 4 ++ src/server/include/pdc_client_server_common.h | 28 ++++++++----- src/server/include/pdc_server_metadata.h | 2 +- src/server/pdc_client_server_common.c | 10 ++--- src/server/pdc_server.c | 14 +++---- .../include/pdc_server_data.h | 2 +- .../pdc_server_region/pdc_server_data.c | 29 ++++++++------ src/tests/CMakeLists.txt | 3 ++ src/tests/kvtag_add_get.c | 4 +- src/tests/read_obj_shared.c | 4 +- src/tests/region_transfer_query.c | 4 +- src/tests/vpicio_mts.c | 10 ++--- 21 files changed, 176 insertions(+), 103 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e7b65d94..8e7727e69 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,6 +131,46 @@ if("${PDC_SOURCE_DIR}" STREQUAL "${PDC_BINARY_DIR}") "Please create a separate binary directory and run CMake there.") endif() +#------------------------------------------------------------------------------ +# Set whether or not to disable compiler warnings +#------------------------------------------------------------------------------ +set(SUPPRESSED_LIST "") +# Disable warnings for unused parameters +option(PDC_SUPPRESS_UNUSED_ARG "Disable unused argument warnings" ON) +if(PDC_SUPPRESS_UNUSED_ARG) + set(SUPPRESSED_LIST "-Wno-unused-parameter" ${SUPPRESSED_LIST}) +endif() + +# Disable warnings for unused variables +option(PDC_SUPPRESS_UNUSED_VAR "Disable unused variable warnings" ON) +if(PDC_SUPPRESS_UNUSED_VAR) + set(SUPPRESSED_LIST "-Wno-unused-variable" ${SUPPRESSED_LIST}) +endif() + +# Disable warnings for unused functions +option(PDC_SUPPRESS_UNUSED_FUNC "Disable unused function warnings" ON) +if(PDC_SUPPRESS_UNUSED_FUNC) + set(SUPPRESSED_LIST "-Wno-unused-function" ${SUPPRESSED_LIST}) +endif() + +# Disable warnings for -Wunused-result +option(PDC_SUPPRESS_UNUSED_RESULT "Disable unused result warnings" ON) +if(PDC_SUPPRESS_UNUSED_RESULT) + set(SUPPRESSED_LIST "-Wno-unused-result" ${SUPPRESSED_LIST}) +endif() + +# Disable warnings for -Wmaybe-uninitialized +option(PDC_SUPPRESS_MAYBE_UNINIT "Disable maybe-uninitialized warnings" ON) +if(PDC_SUPPRESS_MAYBE_UNINIT) + set(SUPPRESSED_LIST "-Wno-maybe-uninitialized" ${SUPPRESSED_LIST}) +endif() + +# Disable warnings for -Wunused-but-set-variable +option(PDC_SUPPRESS_UNUSED_BUT_SET "Disable unused-but-set-variable warnings" ON) +if(PDC_SUPPRESS_UNUSED_BUT_SET) + set(SUPPRESSED_LIST "-Wno-unused-but-set-variable" ${SUPPRESSED_LIST}) +endif() + #------------------------------------------------------------------------------ # Set a default build type if none was specified #------------------------------------------------------------------------------ diff --git a/README.md b/README.md index 54ab218f6..5c5e6cbec 100644 --- a/README.md +++ b/README.md @@ -133,13 +133,16 @@ Now, it's time to compile and install PDC. * One can replace `mpicc` to other available MPI compilers. For example, on Cori, `cc` can be used to replace `mpicc`. * `ctest` contains both sequential and MPI tests for the PDC settings. These can be used to perform regression tests. +* Make sure MERCURY_HOME is added to CMAKE_PREFIX_PATH or PATH. + ```bash cd $PDC_SRC_DIR git checkout develop mkdir build cd build -cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=cc -DMPI_RUN_CMD=srun + +cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_PREFIX_PATH=$MERCURY_DIR -DCMAKE_C_COMPILER=cc -DMPI_RUN_CMD=srun make -j 32 && make install ``` diff --git a/scripts/llsm_importer/gen_script.sh b/scripts/llsm_importer/gen_script.sh index efefd819f..ffc5b6d3b 100755 --- a/scripts/llsm_importer/gen_script.sh +++ b/scripts/llsm_importer/gen_script.sh @@ -6,14 +6,14 @@ MAX_HYPERTHREADING=2 # Designated number of threads per process on each node # (this should be associated with -c option in srun) -NUM_THREAD_PER_SERVER_PROC=128 +NUM_THREAD_PER_SERVER_PROC=64 NUM_THREAD_PER_CLIENT_PROC=64 # Designated number of processes for server anc client on each node # (this should be associated with -n option in srun) NUM_SERVER_PROC_PER_NODE=1 -NUM_CLIENT_PROC_PER_NODE=2 +NUM_CLIENT_PROC_PER_NODE=1 MAX_NODE=512 @@ -35,7 +35,7 @@ for (( i = 1; i <= $MAX_NODE; i*=2 )); do sed -i "s/N_CLIENT_PROC/${NUM_CLIENT_PROC_PER_NODE}/g" $TARGET sed -i "s/NTHREAD_PER_SPROC/${NUM_THREAD_PER_SERVER_PROC}/g" $TARGET sed -i "s/NTHREAD_PER_CPROC/${NUM_THREAD_PER_CLIENT_PROC}/g" $TARGET - if [[ "$i" -gt "16" ]]; then + if [[ "$i" -gt "4" ]]; then sed -i "s/REG//g" $TARGET else sed -i "s/DBG//g" $TARGET diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index 1ba4b8389..cfbb8ac21 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -107,6 +107,10 @@ message(STATUS "PDC_BUILD_INCLUDE_DEPENDENCIES: ${PDC_BUILD_INCLUDE_DEPENDENCIES add_library(pdc ${PDC_SRCS}) +if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(pdc PRIVATE ${SUPPRESSED_LIST}) +endif() + target_include_directories(pdc PUBLIC "$" $ @@ -129,6 +133,11 @@ set(PDC_EXPORTED_LIBS pdc ${PDC_EXPORTED_LIBS}) add_executable(close_server close_server.c ) + +if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(close_server PRIVATE ${SUPPRESSED_LIST}) +endif() + target_link_libraries(close_server pdc ${PDC_COMMON_LIBRARIES}) install( diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index fff47f027..467de4bca 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -1126,8 +1126,8 @@ perr_t PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int port) { perr_t ret_value = SUCCEED; - char na_info_string[PATH_MAX]; - char hostname[ADDR_MAX]; + char na_info_string[NA_STRING_INFO_LEN]; + char hostname[HOSTNAME_LEN]; int local_server_id; /* Set the default mercury transport * but enable overriding that to any of: diff --git a/src/commons/CMakeLists.txt b/src/commons/CMakeLists.txt index 9f8abf690..485ec4b8a 100644 --- a/src/commons/CMakeLists.txt +++ b/src/commons/CMakeLists.txt @@ -22,6 +22,8 @@ endif() find_package(MERCURY REQUIRED) if(MERCURY_FOUND) message(STATUS "mercury dir = ${MERCURY_DIR}") + # NOTE: enable the following if you need ${MERCURY_INCLUDE_DIR} in the future + # NOTE: remember to add MERCURY_HOME to PATH or CMAKE_PREFIX_PATH if you enable the following. find_path(MERCURY_INCLUDE_DIR mercury.h HINTS ${MERCURY_DIR}) find_library(MERCURY_LIBRARY mercury HINTS ${MERCURY_DIR}) message(STATUS "mercury include dir = ${MERCURY_INCLUDE_DIR}") @@ -103,6 +105,10 @@ file(GLOB_RECURSE PDC_COMMONS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.h) add_library(${PDC_COMMON_LIBRARY_NAME} ${PDC_LIBTYPE} ${PDC_COMMONS_SOURCES} ${PDC_COMMONS_HEADERS}) +if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${PDC_COMMON_LIBRARY_NAME} PRIVATE ${SUPPRESSED_LIST}) +endif() + target_include_directories(${PDC_COMMON_LIBRARY_NAME} PUBLIC "$" $ @@ -207,4 +213,4 @@ set(PDC_INCLUDES_INSTALL_TIME PARENT_SCOPE ) -set(PDC_COMMONS_LIBRARIES ${PDC_COMMON_LIBRARY_NAME} PARENT_SCOPE) \ No newline at end of file +set(PDC_COMMONS_LIBRARIES ${PDC_COMMON_LIBRARY_NAME} PARENT_SCOPE) diff --git a/src/commons/generic/include/pdc_generic.h b/src/commons/generic/include/pdc_generic.h index 47322b553..fc9925ea9 100644 --- a/src/commons/generic/include/pdc_generic.h +++ b/src/commons/generic/include/pdc_generic.h @@ -17,7 +17,7 @@ typedef enum { false = 0, true = 1 } bool; #endif #endif -typedef enum { +typedef enum pdc_c_var_type_t { PDC_UNKNOWN = -1, /* error */ PDC_INT = 0, /* integer types (identical to int32_t) */ PDC_FLOAT = 1, /* floating-point types */ @@ -41,9 +41,9 @@ typedef enum { PDC_TYPE_COUNT = 19 /* this is the number of var types and has to be the last */ } pdc_c_var_type_t; -typedef pdc_c_var_type_t PDC_CType; +// typedef pdc_c_var_type_t PDC_CType; -typedef enum { +typedef enum pdc_c_var_class_t { PDC_CLS_SCALAR, PDC_CLS_ARRAY, PDC_CLS_ENUM, // not implemented, users can use PDC_CT_INT @@ -57,10 +57,10 @@ typedef enum { PDC_CLS_COUNT // just the count of the enum. } pdc_c_var_class_t; -typedef pdc_c_var_class_t PDC_CType_Class; +// typedef pdc_c_var_class_t PDC_CType_Class; // clang-format off -static const size_t DataTypeSizes[PDC_TYPE_COUNT] = { +static size_t DataTypeSizes[PDC_TYPE_COUNT] = { sizeof(int), sizeof(float), sizeof(double), @@ -82,7 +82,7 @@ static const size_t DataTypeSizes[PDC_TYPE_COUNT] = { sizeof(size_t) }; -static const char *DataTypeNames[PDC_TYPE_COUNT] = { +static char *DataTypeNames[PDC_TYPE_COUNT] = { "int", "float", "double", @@ -104,7 +104,7 @@ static const char *DataTypeNames[PDC_TYPE_COUNT] = { "size_t" }; -static const char *DataTypeEnumNames[PDC_TYPE_COUNT] = { +static char *DataTypeEnumNames[PDC_TYPE_COUNT] = { "PDC_INT", "PDC_FLOAT", "PDC_DOUBLE", @@ -126,7 +126,8 @@ static const char *DataTypeEnumNames[PDC_TYPE_COUNT] = { "PDC_SIZE_T" }; -static const char *DataTypeFormat[PDC_TYPE_COUNT] = { +__attribute__((unused)) +static char *DataTypeFormat[PDC_TYPE_COUNT] = { "%d", // int "%f", // float "%lf", // double @@ -149,8 +150,7 @@ static const char *DataTypeFormat[PDC_TYPE_COUNT] = { }; // clang-format on - -static const char * +__attribute__((unused)) static char * get_enum_name_by_dtype(pdc_c_var_type_t type) { if (type < 0 || type >= PDC_TYPE_COUNT) { @@ -158,8 +158,7 @@ get_enum_name_by_dtype(pdc_c_var_type_t type) } return DataTypeEnumNames[type]; } - -static const size_t +__attribute__((unused)) static size_t get_size_by_dtype(pdc_c_var_type_t type) { if (type < 0 || type >= PDC_TYPE_COUNT) { @@ -167,8 +166,7 @@ get_size_by_dtype(pdc_c_var_type_t type) } return DataTypeSizes[type]; } - -static const size_t +__attribute__((unused)) static size_t get_size_by_class_n_type(void *data, size_t item_count, pdc_c_var_class_t pdc_class, pdc_c_var_type_t pdc_type) { @@ -184,7 +182,7 @@ get_size_by_class_n_type(void *data, size_t item_count, pdc_c_var_class_t pdc_cl else if (pdc_class == PDC_CLS_ARRAY) { if (pdc_type == PDC_STRING) { char **str_arr = (char **)data; - int i = 0; + size_t i = 0; for (i = 0; i < item_count; i++) { size = size + (strlen(str_arr[i]) + 1) * sizeof(char); } @@ -195,8 +193,7 @@ get_size_by_class_n_type(void *data, size_t item_count, pdc_c_var_class_t pdc_cl } return size; } - -static const char * +__attribute__((unused)) static char * get_name_by_dtype(pdc_c_var_type_t type) { if (type < 0 || type >= PDC_TYPE_COUNT) { @@ -204,8 +201,7 @@ get_name_by_dtype(pdc_c_var_type_t type) } return DataTypeNames[type]; } - -static pdc_c_var_type_t +__attribute__((unused)) static pdc_c_var_type_t get_dtype_by_enum_name(const char *enumName) { for (int i = 0; i < PDC_TYPE_COUNT; i++) { @@ -216,4 +212,4 @@ get_dtype_by_enum_name(const char *enumName) return PDC_UNKNOWN; // assuming PDC_UNKNOWN is the enum value for "unknown" } -#endif /* PDC_GENERIC_H */ \ No newline at end of file +#endif /* PDC_GENERIC_H */ diff --git a/src/commons/serde/include/pdc_serde.h b/src/commons/serde/include/pdc_serde.h index 8c5efae9e..c65c70b33 100644 --- a/src/commons/serde/include/pdc_serde.h +++ b/src/commons/serde/include/pdc_serde.h @@ -10,18 +10,18 @@ #define MAX_BUFFER_SIZE 1000 typedef struct { - PDC_CType pdc_type; /**< Data type of the key */ - size_t size; /**< Size of the key */ - void * key; /**< Pointer to the key data */ + pdc_c_var_type_t pdc_type; /**< Data type of the key */ + size_t size; /**< Size of the key */ + void * key; /**< Pointer to the key data */ } PDC_SERDE_Key; typedef struct { - PDC_CType_Class pdc_class; /**< Class of the value */ - PDC_CType pdc_type; /**< Data type of the value */ - size_t size; // size of the data. If a string, it is strlen(data) + 1; - // if an array, it is the number of elements; - // if a struct, it is the totalSize of the data chunk of the struct, etc. - void *data; /**< Pointer to the value data */ + pdc_c_var_class_t pdc_class; /**< Class of the value */ + pdc_c_var_type_t pdc_type; /**< Data type of the value */ + size_t size; // size of the data. If a string, it is strlen(data) + 1; + // if an array, it is the number of elements; + // if a struct, it is the totalSize of the data chunk of the struct, etc. + void *data; /**< Pointer to the value data */ } PDC_SERDE_Value; typedef struct { @@ -111,7 +111,7 @@ void pdc_serde_print(PDC_SERDE_SerializedData *data); * @return Pointer to the created PDC_SERDE_Key structure */ static inline PDC_SERDE_Key * -PDC_SERDE_KEY(void *key, PDC_CType pdc_type, size_t size) +PDC_SERDE_KEY(void *key, pdc_c_var_type_t pdc_type, size_t size) { PDC_SERDE_Key *pdc_key = (PDC_SERDE_Key *)malloc(sizeof(PDC_SERDE_Key)); size_t key_size = (size_t)get_size_by_class_n_type(key, size, PDC_CLS_SCALAR, pdc_type); @@ -128,12 +128,15 @@ PDC_SERDE_KEY(void *key, PDC_CType pdc_type, size_t size) * @param data Pointer to the value data * @param pdc_type Data type of the value * @param pdc_class Class of the value - * @param size Size of the value data + * @param size Size of the value data. + * For scalar value, it is the result of sizeof(type) function; + * for array, it is the number of elements; + * for struct, it is the totalSize of the data chunk of the struct, etc. * * @return Pointer to the created PDC_SERDE_Value structure */ static inline PDC_SERDE_Value * -PDC_SERDE_VALUE(void *data, PDC_CType pdc_type, PDC_CType_Class pdc_class, size_t size) +PDC_SERDE_VALUE(void *data, pdc_c_var_type_t pdc_type, pdc_c_var_class_t pdc_class, size_t size) { PDC_SERDE_Value *pdc_value = (PDC_SERDE_Value *)malloc(sizeof(PDC_SERDE_Value)); size_t value_size = 0; diff --git a/src/commons/serde/pdc_serde.c b/src/commons/serde/pdc_serde.c index fe9e9aa64..21a9e9967 100644 --- a/src/commons/serde/pdc_serde.c +++ b/src/commons/serde/pdc_serde.c @@ -98,7 +98,7 @@ pdc_serde_serialize(PDC_SERDE_SerializedData *data) memcpy(buffer + sizeof(size_t) * 2, &data->header->numKeys, sizeof(size_t)); // then the keys size_t offset = sizeof(size_t) * 3; - for (int i = 0; i < data->header->numKeys; i++) { + for (size_t i = 0; i < data->header->numKeys; i++) { int8_t pdc_type = (int8_t)(data->header->keys[i].pdc_type); memcpy(buffer + offset, &pdc_type, sizeof(int8_t)); offset += sizeof(int8_t); @@ -117,7 +117,7 @@ pdc_serde_serialize(PDC_SERDE_SerializedData *data) memcpy(buffer + offset, &data->data->numValues, sizeof(size_t)); offset += sizeof(size_t); // then the values - for (int i = 0; i < data->data->numValues; i++) { + for (size_t i = 0; i < data->data->numValues; i++) { int8_t pdc_class = (int8_t)data->data->values[i].pdc_class; int8_t pdc_type = (int8_t)data->data->values[i].pdc_type; memcpy(buffer + offset, &pdc_class, sizeof(int8_t)); @@ -155,7 +155,7 @@ pdc_serde_deserialize(void *buffer) header->keys = malloc(sizeof(PDC_SERDE_Key) * numKeys); header->numKeys = numKeys; header->totalSize = headerSize; - for (int i = 0; i < numKeys; i++) { + for (size_t i = 0; i < numKeys; i++) { int8_t pdc_type; size_t size; memcpy(&pdc_type, buffer + offset, sizeof(int8_t)); @@ -166,7 +166,7 @@ pdc_serde_deserialize(void *buffer) memcpy(key, buffer + offset, size); offset += size; header->keys[i].key = key; - header->keys[i].pdc_type = (PDC_CType)pdc_type; + header->keys[i].pdc_type = (pdc_c_var_type_t)pdc_type; header->keys[i].size = size; } @@ -188,7 +188,7 @@ pdc_serde_deserialize(void *buffer) data->values = malloc(sizeof(PDC_SERDE_Value) * numValues); data->numValues = numValues; data->totalSize = dataSize; - for (int i = 0; i < numValues; i++) { + for (size_t i = 0; i < numValues; i++) { int8_t pdc_class; int8_t pdc_type; size_t size; @@ -202,8 +202,8 @@ pdc_serde_deserialize(void *buffer) memcpy(value, buffer + offset, size); offset += size; data->values[i].data = value; - data->values[i].pdc_class = (PDC_CType_Class)pdc_class; - data->values[i].pdc_type = (PDC_CType)pdc_type; + data->values[i].pdc_class = (pdc_c_var_class_t)pdc_class; + data->values[i].pdc_type = (pdc_c_var_type_t)pdc_type; data->values[i].size = size; } // check the total size @@ -230,11 +230,11 @@ pdc_serde_deserialize(void *buffer) void pdc_serde_free(PDC_SERDE_SerializedData *data) { - for (int i = 0; i < data->header->numKeys; i++) { + for (size_t i = 0; i < data->header->numKeys; i++) { free(data->header->keys[i].key); } free(data->header->keys); - for (int i = 0; i < data->data->numValues; i++) { + for (size_t i = 0; i < data->data->numValues; i++) { free(data->data->values[i].data); } free(data->data->values); @@ -249,8 +249,8 @@ pdc_serde_print(PDC_SERDE_SerializedData *data) printf("Header:\n"); printf(" numKeys: %zu\n", data->header->numKeys); printf(" totalSize: %zu\n", data->header->totalSize); - for (int i = 0; i < data->header->numKeys; i++) { - printf(" key %d:\n", i); + for (size_t i = 0; i < data->header->numKeys; i++) { + printf(" key %ld:\n", i); printf(" type: %d\n", data->header->keys[i].pdc_type); printf(" size: %zu\n", data->header->keys[i].size); printf(" key: %s\n", (char *)data->header->keys[i].key); @@ -258,13 +258,13 @@ pdc_serde_print(PDC_SERDE_SerializedData *data) printf("Data:\n"); printf(" numValues: %zu\n", data->data->numValues); printf(" totalSize: %zu\n", data->data->totalSize); - for (int i = 0; i < data->data->numValues; i++) { - printf(" value %d:\n", i); + for (size_t i = 0; i < data->data->numValues; i++) { + printf(" value %ld:\n", i); printf(" class: %d\n", data->data->values[i].pdc_class); printf(" type: %d\n", data->data->values[i].pdc_type); printf(" size: %zu\n", data->data->values[i].size); printf(" data: "); - if (data->data->values[i].pdc_class == PDC_STRING) { + if (data->data->values[i].pdc_type == PDC_STRING) { printf("%s\n", (char *)data->data->values[i].data); } else { @@ -302,7 +302,7 @@ test_serde_framework() char * arrayKey_str = "array"; int intArray[3] = {1, 2, 3}; PDC_SERDE_Key * arrayKey = PDC_SERDE_KEY(arrayKey_str, PDC_STRING, sizeof(arrayKey_str)); - PDC_SERDE_Value *arrayValue = PDC_SERDE_VALUE(intArray, PDC_INT, PDC_CLS_ARRAY, sizeof(int) * 3); + PDC_SERDE_Value *arrayValue = PDC_SERDE_VALUE(intArray, PDC_INT, PDC_CLS_ARRAY, 3); pdc_serde_append_key_value(data, arrayKey, arrayValue); typedef struct { diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index abe3917c6..5d73b6b12 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -53,6 +53,10 @@ add_executable(pdc_server.exe ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_hist_pkg.c ) +if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(pdc_server.exe PRIVATE ${SUPPRESSED_LIST}) +endif() + #install( # TARGETS # pdc_server.exe diff --git a/src/server/include/pdc_client_server_common.h b/src/server/include/pdc_client_server_common.h index e125af9a5..e3ea447ea 100644 --- a/src/server/include/pdc_client_server_common.h +++ b/src/server/include/pdc_client_server_common.h @@ -57,18 +57,14 @@ hg_thread_mutex_t meta_buf_map_mutex_g; hg_thread_mutex_t meta_obj_map_mutex_g; #endif -#ifndef HOST_NAME_MAX -#if defined(__APPLE__) -#define HOST_NAME_MAX 255 -#else -#define HOST_NAME_MAX 64 -#endif /* __APPLE__ */ -#endif /* HOST_NAME_MAX */ - #define PAGE_SIZE 4096 -#define ADDR_MAX 512 +#define ADDR_MAX 1024 +#define NA_STRING_INFO_LEN ADDR_MAX / 2 +#define HOSTNAME_LEN ADDR_MAX / 8 +#define TMP_DIR_STRING_LEN ADDR_MAX / 2 #define DIM_MAX 4 #define TAG_LEN_MAX 2048 +#define OBJ_NAME_MAX TAG_LEN_MAX / 2 #define PDC_SERVER_ID_INTERVEL 1000000000ull #define PDC_SERVER_MAX_PROC_PER_NODE 32 #define PDC_SERIALIZE_MAX_SIZE 256 @@ -79,6 +75,16 @@ hg_thread_mutex_t meta_obj_map_mutex_g; #define PDC_UPDATE_CACHE 111 #define PDC_UPDATE_STORAGE 101 +#ifndef HOST_NAME_MAX +#if defined(__APPLE__) +#define HOST_NAME_MAX ADDR_MAX / 4 +#define HOSTNAME_LEN HOST_NAME_MAX +#else +#define HOST_NAME_MAX ADDR_MAX / 8 +#define HOSTNAME_LEN HOST_NAME_MAX +#endif /* __APPLE__ */ +#endif /* HOST_NAME_MAX */ + #define pdc_server_cfg_name_g "server.cfg" #define ADD_OBJ 1 @@ -391,8 +397,8 @@ typedef struct data_server_region_unmap_t { // For storing metadata typedef struct pdc_metadata_t { int user_id; // Both server and client gets it and do security check - char app_name[ADDR_MAX]; - char obj_name[ADDR_MAX]; + char app_name[OBJ_NAME_MAX]; + char obj_name[OBJ_NAME_MAX]; int time_step; // Above four are the unique identifier for objects diff --git a/src/server/include/pdc_server_metadata.h b/src/server/include/pdc_server_metadata.h index f81225a7d..c65ff59bc 100644 --- a/src/server/include/pdc_server_metadata.h +++ b/src/server/include/pdc_server_metadata.h @@ -44,7 +44,7 @@ /*****************************/ extern int pdc_server_rank_g; extern int pdc_server_size_g; -extern char pdc_server_tmp_dir_g[ADDR_MAX]; +extern char pdc_server_tmp_dir_g[TMP_DIR_STRING_LEN]; extern uint32_t n_metadata_g; extern HashTable * metadata_hash_table_g; extern HashTable * container_hash_table_g; diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index 859ce7bb8..e3e8597b1 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -401,7 +401,7 @@ PDC_metadata_cmp(pdc_metadata_t *a, pdc_metadata_t *b) void PDC_mkdir(const char *dir) { - char tmp[ADDR_MAX]; + char tmp[TMP_DIR_STRING_LEN]; char *p = NULL; FUNC_ENTER(NULL); @@ -476,8 +476,8 @@ PDC_metadata_init(pdc_metadata_t *a) a->ndim = 0; a->data_server_id = 0; - memset(a->app_name, 0, sizeof(char) * ADDR_MAX); - memset(a->obj_name, 0, sizeof(char) * ADDR_MAX); + memset(a->app_name, 0, sizeof(char) * NAME_MAX); + memset(a->obj_name, 0, sizeof(char) * NAME_MAX); memset(a->tags, 0, sizeof(char) * TAG_LEN_MAX); memset(a->data_location, 0, sizeof(char) * ADDR_MAX); memset(a->dims, 0, sizeof(uint64_t) * DIM_MAX); @@ -3096,7 +3096,7 @@ HG_TEST_RPC_CB(region_release, handle) size2 = HG_Bulk_get_size(remote_bulk_handle); if (size != size2) { error = 1; - printf("==PDC_SERVER: local size %llu, remote %llu\n", size, size2); + printf("==PDC_SERVER: local size %lu, remote %lu\n", size, size2); PGOTO_ERROR(HG_OTHER_ERROR, "===PDC SERVER: HG_TEST_RPC_CB(region_release, " "handle) local and remote bulk size does not match"); } @@ -3269,7 +3269,7 @@ HG_TEST_RPC_CB(region_release, handle) size2 = HG_Bulk_get_size(remote_bulk_handle); if (size != size2) { error = 1; - printf("==PDC_SERVER: local size %llu, remote %llu\n", size, size2); + printf("==PDC_SERVER: local size %lu, remote %lu\n", size, size2); /* PGOTO_ERROR(HG_OTHER_ERROR, "===PDC SERVER: HG_TEST_RPC_CB(region_release, * handle) local and remote bulk size does not match"); */ } diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index e08882120..f461c31d5 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -111,7 +111,7 @@ extern hg_thread_pool_t *hg_test_thread_pool_g; extern hg_thread_pool_t *hg_test_thread_pool_fs_g; hg_atomic_int32_t close_server_g; -char pdc_server_tmp_dir_g[ADDR_MAX]; +char pdc_server_tmp_dir_g[TMP_DIR_STRING_LEN]; int is_restart_g = 0; int pdc_server_rank_g = 0; int pdc_server_size_g = 1; @@ -693,8 +693,8 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) perr_t ret_value = SUCCEED; int i = 0; char self_addr_string[ADDR_MAX]; - char na_info_string[ADDR_MAX]; - char hostname[1024]; + char na_info_string[NA_STRING_INFO_LEN]; + char hostname[HOSTNAME_LEN]; struct hg_init_info init_info = {0}; /* Set the default mercury transport @@ -728,9 +728,9 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) if ((hg_transport = getenv("HG_TRANSPORT")) == NULL) { hg_transport = default_hg_transport; } - memset(hostname, 0, 1024); - gethostname(hostname, 1023); - snprintf(na_info_string, ADDR_MAX, "%s://%s:%d", hg_transport, hostname, port); + memset(hostname, 0, HOSTNAME_LEN); + gethostname(hostname, HOSTNAME_LEN - 1); + snprintf(na_info_string, NA_STRING_INFO_LEN, "%s://%s:%d", hg_transport, hostname, port); if (pdc_server_rank_g == 0) printf("==PDC_SERVER[%d]: using %.7s\n", pdc_server_rank_g, na_info_string); @@ -1990,7 +1990,7 @@ PDC_Server_get_env() if (tmp_env_char == NULL) tmp_env_char = "./pdc_tmp"; - snprintf(pdc_server_tmp_dir_g, ADDR_MAX, "%s/", tmp_env_char); + snprintf(pdc_server_tmp_dir_g, TMP_DIR_STRING_LEN, "%s/", tmp_env_char); lustre_total_ost_g = 1; #ifdef ENABLE_LUSTRE diff --git a/src/server/pdc_server_region/include/pdc_server_data.h b/src/server/pdc_server_region/include/pdc_server_data.h index 4bcdac9dd..a204aa437 100644 --- a/src/server/pdc_server_region/include/pdc_server_data.h +++ b/src/server/pdc_server_region/include/pdc_server_data.h @@ -244,7 +244,7 @@ typedef struct cache_storage_region_t { /*****************************/ extern int pdc_server_rank_g; extern int pdc_server_size_g; -extern char pdc_server_tmp_dir_g[ADDR_MAX]; +extern char pdc_server_tmp_dir_g[TMP_DIR_STRING_LEN]; extern double server_write_time_g; extern double server_read_time_g; extern double server_get_storage_info_time_g; diff --git a/src/server/pdc_server_region/pdc_server_data.c b/src/server/pdc_server_region/pdc_server_data.c index 6f7974b5d..12747ee5c 100644 --- a/src/server/pdc_server_region/pdc_server_data.c +++ b/src/server/pdc_server_region/pdc_server_data.c @@ -130,7 +130,7 @@ PDC_Server_set_lustre_stripe(const char *path, int stripe_count, int stripe_size perr_t ret_value = SUCCEED; size_t len; int i, index; - char tmp[ADDR_MAX]; + char tmp[TMP_DIR_STRING_LEN]; char cmd[TAG_LEN_MAX]; FUNC_ENTER(NULL); @@ -4695,7 +4695,7 @@ PDC_Server_posix_write(int fd, void *buf, uint64_t write_size) while (write_size > max_write_size) { ret = write(fd, buf, max_write_size); if (ret < 0 || ret != (ssize_t)max_write_size) { - printf("==PDC_SERVER[%d]: in-loop: write %d failed, ret = %d, max_write_size = %llu\n", + printf("==PDC_SERVER[%d]: in-loop: write %d failed, ret = %ld, max_write_size = %lu\n", pdc_server_rank_g, fd, ret, max_write_size); ret_value = FAIL; goto done; @@ -4737,7 +4737,7 @@ PDC_Server_data_write_out(uint64_t obj_id, struct pdc_region_info *region_info, double start = MPI_Wtime(), start_posix; #endif - uint64_t write_size; + uint64_t write_size = 0; if (region_info->ndim >= 1) write_size = unit * region_info->size[0]; if (region_info->ndim >= 2) @@ -7253,15 +7253,18 @@ PDC_Server_query_evaluate_merge_opt(pdc_query_t *query, query_task_t *task, pdc_ pdc_selection_t *sel = query->sel; uint64_t nelem; size_t i, j, unit_size; - pdc_query_op_t op = PDC_QUERY_OR, lop = PDC_QUERY_OR, rop = PDC_QUERY_OR; - float flo = .0, fhi = .0; - double dlo = .0, dhi = .0; - int ilo = 0, ihi = 0, ndim, count = 0; - uint32_t ulo = 0, uhi = 0; - int64_t i64lo = 0, i64hi = 0; - uint64_t ui64lo = 0, ui64hi = 0; - void * value = NULL, *buf = NULL; - int n_eval_region = 0, can_skip, region_iter = 0; + // FIXME: need to check the types of these 'op's. I think they should be of the following (or don't even + // need to be initilized): + pdc_query_op_t op = PDC_EQ, lop = PDC_EQ, rop = PDC_EQ; + // pdc_query_op_t op = PDC_QUERY_OR, lop = PDC_QUERY_OR, rop = PDC_QUERY_OR; + float flo = .0, fhi = .0; + double dlo = .0, dhi = .0; + int ilo = 0, ihi = 0, ndim, count = 0; + uint32_t ulo = 0, uhi = 0; + int64_t i64lo = 0, i64hi = 0; + uint64_t ui64lo = 0, ui64hi = 0; + void * value = NULL, *buf = NULL; + int n_eval_region = 0, can_skip, region_iter = 0; printf("==PDC_SERVER[%d]: %s - start query evaluation!\n", pdc_server_rank_g, __func__); fflush(stdout); @@ -9484,7 +9487,7 @@ PDC_Server_recv_get_sel_data(const struct hg_cb_info *callback_info) get_sel_data_rpc_in_t *in = (get_sel_data_rpc_in_t *)callback_info->arg; query_task_t * task_elt, *task = NULL; pdc_metadata_t * meta; - struct hg_cb_info fake_callback_info; + struct hg_cb_info fake_callback_info = {0}; DL_FOREACH(query_task_list_head_g, task_elt) { diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 499421880..2fd2d0839 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -107,6 +107,9 @@ set(PROGRAMS foreach(program ${PROGRAMS}) add_executable(${program} ${program}.c) + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${program} PRIVATE ${SUPPRESSED_LIST}) + endif() target_link_libraries(${program} pdc) endforeach(program) diff --git a/src/tests/kvtag_add_get.c b/src/tests/kvtag_add_get.c index 91686b9be..7d53da6a6 100644 --- a/src/tests/kvtag_add_get.c +++ b/src/tests/kvtag_add_get.c @@ -94,12 +94,12 @@ main() kvtag2.name = "key2int"; kvtag2.value = (void *)&v2; - kvtag1.type = PDC_INT; + kvtag2.type = PDC_INT; kvtag2.size = sizeof(int); kvtag3.name = "key3double"; kvtag3.value = (void *)&v3; - kvtag1.type = PDC_DOUBLE; + kvtag3.type = PDC_DOUBLE; kvtag3.size = sizeof(double); if (PDCobj_put_tag(obj1, kvtag1.name, kvtag1.value, kvtag1.type, kvtag1.size) < 0) diff --git a/src/tests/read_obj_shared.c b/src/tests/read_obj_shared.c index 6f56965d8..4b0058dbd 100644 --- a/src/tests/read_obj_shared.c +++ b/src/tests/read_obj_shared.c @@ -163,7 +163,7 @@ main(int argc, char **argv) offset[0] = rank * my_data_size; local_offset[0] = 0; mysize[0] = my_data_size; - printf("rank %d offset = %llu, length = %llu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); + printf("rank %d offset = %lu, length = %lu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); local_region = PDCregion_create(ndim, local_offset, mysize); global_region = PDCregion_create(ndim, offset, mysize); @@ -220,7 +220,7 @@ main(int argc, char **argv) offset[0] = rank * my_data_size; local_offset[0] = 0; mysize[0] = my_data_size; - printf("rank %d offset = %llu, length = %llu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); + printf("rank %d offset = %lu, length = %lu, unit size = %ld\n", rank, offset[0], mysize[0], type_size); local_region = PDCregion_create(ndim, local_offset, mysize); global_region = PDCregion_create(ndim, offset, mysize); diff --git a/src/tests/region_transfer_query.c b/src/tests/region_transfer_query.c index e2c7dcfe6..dede54aaa 100644 --- a/src/tests/region_transfer_query.c +++ b/src/tests/region_transfer_query.c @@ -45,9 +45,9 @@ main(int argc, char **argv) int rank = 0, size = 1, i; int ret_value = 0; - uint64_t offset[3], offset_length[3], local_offset[1]; + uint64_t offset[3], offset_length[3]; //, local_offset[1]; uint64_t dims[1]; - local_offset[0] = 0; + // local_offset[0] = 0; offset[0] = 0; offset[1] = 2; offset[2] = 5; diff --git a/src/tests/vpicio_mts.c b/src/tests/vpicio_mts.c index 65b7b4dc8..c80d11c03 100644 --- a/src/tests/vpicio_mts.c +++ b/src/tests/vpicio_mts.c @@ -76,7 +76,7 @@ main(int argc, char **argv) uint64_t *offset_remote; uint64_t *mysize; double t0, t1; - int steps = 1, sleeptime = 0; + uint64_t steps = 1, sleeptime = 0; pdcid_t transfer_request_x, transfer_request_y, transfer_request_z, transfer_request_px, transfer_request_py, transfer_request_pz, transfer_request_id1, transfer_request_id2; @@ -95,8 +95,8 @@ main(int argc, char **argv) sleeptime = atoi(argv[3]); } if (rank == 0) - printf("Writing %" PRIu64 " number of particles for %d steps with %d clients.\n", numparticles, steps, - size); + printf("Writing %" PRIu64 " number of particles for %ld steps with %d clients.\n", numparticles, + steps, size); dims[0] = numparticles; @@ -199,7 +199,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) { - printf("\n#Step %llu\n", i); + printf("\n#Step %lu\n", i); } #endif PDCprop_set_obj_time_step(obj_prop_xx, i); @@ -504,7 +504,7 @@ main(int argc, char **argv) if (i != steps - 1) { sleep(sleeptime); if (rank == 0) { - printf("Sleep time: %d.00\n", sleeptime); + printf("Sleep time: %ld.00\n", sleeptime); } } } // End for steps From af02e66a4bad87d92254bce76b0241f4b12bdcdb Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Fri, 23 Jun 2023 14:29:16 -0700 Subject: [PATCH 199/216] Update clang-format-fix.yml --- .github/workflows/clang-format-fix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clang-format-fix.yml b/.github/workflows/clang-format-fix.yml index 9145f65ca..5ca34c945 100644 --- a/.github/workflows/clang-format-fix.yml +++ b/.github/workflows/clang-format-fix.yml @@ -1,15 +1,15 @@ -name: clang-format Check +name: clang-format autofix on: workflow_dispatch: push: jobs: formatting-check: - name: Formatting Check + name: Formatting Fix runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'skip-ci')" steps: - uses: actions/checkout@v2 - - name: Run clang-format style check for C programs. + - name: Run clang-format style fix for C programs. uses: DoozyX/clang-format-lint-action@v0.11 with: source: '.' From a7e7ecfb02fe38ecc64aa4244a29f662ebe0f58a Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Fri, 23 Jun 2023 14:33:43 -0700 Subject: [PATCH 200/216] Update clang-format-fix.yml --- .github/workflows/clang-format-fix.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/clang-format-fix.yml b/.github/workflows/clang-format-fix.yml index 5ca34c945..686218650 100644 --- a/.github/workflows/clang-format-fix.yml +++ b/.github/workflows/clang-format-fix.yml @@ -1,16 +1,18 @@ -name: clang-format autofix +name: clang-format Commit Chanages on: workflow_dispatch: push: jobs: formatting-check: - name: Formatting Fix + name: Commit Format Changes runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'skip-ci')" + permissions: + contents: write # In order to allow EndBug/add-and-commit to commit changes steps: - - uses: actions/checkout@v2 - - name: Run clang-format style fix for C programs. - uses: DoozyX/clang-format-lint-action@v0.11 + - uses: actions/checkout@v3 + - name: Fix C formatting issues detected by clang-format + uses: DoozyX/clang-format-lint-action@v0.13 with: source: '.' extensions: 'c,h,cpp,hpp' @@ -18,7 +20,7 @@ jobs: inplace: True style: file # exclude: './config ' - - uses: EndBug/add-and-commit@v7 + - uses: EndBug/add-and-commit@v9 with: author_name: github-actions author_email: github-actions[bot]@users.noreply.github.com From cd5b3057399bfcb66476103fd8a6add1445e60d5 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Thu, 29 Jun 2023 15:24:03 -0700 Subject: [PATCH 201/216] Increase the default server cache size to 32GB and flush frequency to 30s --- src/server/pdc_server_region/pdc_server_region_cache.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/pdc_server_region/pdc_server_region_cache.c b/src/server/pdc_server_region/pdc_server_region_cache.c index 6bcf5217b..5414eff85 100644 --- a/src/server/pdc_server_region/pdc_server_region_cache.c +++ b/src/server/pdc_server_region/pdc_server_region_cache.c @@ -3,7 +3,8 @@ #ifdef PDC_SERVER_CACHE -#define MAX_CACHE_SIZE 1610612736 +#define MAX_CACHE_SIZE 34359738368 +#define PDC_CACHE_FLUSH_TIME_INT 30 typedef struct pdc_region_cache { struct pdc_region_info * region_cache_info; @@ -825,7 +826,7 @@ PDC_region_cache_clock_cycle(void *ptr) struct timeval current_time; struct timeval finish_time; int nflush = 0; - double flush_frequency_s = 2.0, elapsed_time; + double flush_frequency_s = PDC_CACHE_FLUSH_TIME_INT, elapsed_time; int server_rank = 0; char *p = getenv("PDC_SERVER_CACHE_FLUSH_FREQUENCY_S"); From 1578f00f6c259a35f42144d154de49844beffc61 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 29 Jun 2023 22:24:41 +0000 Subject: [PATCH 202/216] Committing clang-format changes --- src/server/pdc_server_region/pdc_server_region_cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/pdc_server_region/pdc_server_region_cache.c b/src/server/pdc_server_region/pdc_server_region_cache.c index 5414eff85..92c408fa6 100644 --- a/src/server/pdc_server_region/pdc_server_region_cache.c +++ b/src/server/pdc_server_region/pdc_server_region_cache.c @@ -3,7 +3,7 @@ #ifdef PDC_SERVER_CACHE -#define MAX_CACHE_SIZE 34359738368 +#define MAX_CACHE_SIZE 34359738368 #define PDC_CACHE_FLUSH_TIME_INT 30 typedef struct pdc_region_cache { From 10189bf20ddafb714b6f91b4d29a2a8b50444ffa Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 10 Jul 2023 15:41:24 -0400 Subject: [PATCH 203/216] add FindMERCURY.cmake --- CMake/FindMERCURY.cmake | 46 +++++++++++++++++++++++++++++++++++++++++ CMakeLists.txt | 25 ++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 CMake/FindMERCURY.cmake diff --git a/CMake/FindMERCURY.cmake b/CMake/FindMERCURY.cmake new file mode 100644 index 000000000..869d9a647 --- /dev/null +++ b/CMake/FindMERCURY.cmake @@ -0,0 +1,46 @@ +# FindMERCURY.cmake + +# Find the system's MERCURY library +# This will define: +# +# MERCURY_FOUND - System has MERCURY +# MERCURY_INCLUDE_DIRS - The MERCURY include directory +# MERCURY_LIBRARIES - The libraries needed to use MERCURY + +find_package(MERCURY QUIET HINTS $ENV{MERCURY_HOME} $ENV{MERCURY_DIR} $ENV{MERCURY_ROOT} $ENV{MERCURYPATH} $ENV{MERCURY_PATH}) +if(MERCURY_FOUND) + message(STATUS "mercury dir = ${MERCURY_DIR}") + # NOTE: enable the following if you need ${MERCURY_INCLUDE_DIR} in the future + # NOTE: remember to add MERCURY_HOME to PATH or CMAKE_PREFIX_PATH if you enable the following. + find_path(MERCURY_INCLUDE_DIR mercury.h HINTS ${MERCURY_DIR}) + find_library(MERCURY_LIBRARY + NAMES + mercury + mercury_debug + HINTS ${MERCURY_DIR} + ) + + find_library(MERCURY_NA_LIBRARY + NAMES + na + na_debug + HINTS ${MERCURY_DIR} + ) + + find_library(MERCURY_UTIL_LIBRARY + NAMES + mercury_util + HINTS ${MERCURY_DIR} + ) + + set(MERCURY_LIBRARIES ${MERCURY_LIBRARY} ${MERCURY_NA_LIBRARY} ${MERCURY_UTIL_LIBRARY}) + set(MERCURY_INCLUDE_DIRS ${MERCURY_INCLUDE_DIR}) + message(STATUS "mercury include dir = ${MERCURY_INCLUDE_DIRS}") + message(STATUS "mercury lib = ${MERCURY_LIBRARIES}") + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(MERCURY DEFAULT_MSG MERCURY_LIBRARY MERCURY_INCLUDE_DIR) +else(MERCURY_FOUND) + set(MERCURY_LIBRARIES "") +endif() + +mark_as_advanced(MERCURY_INCLUDE_DIR MERCURY_LIBRARY) \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e7b65d94..d878252fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,18 @@ set(PDC_PACKAGE_TARNAME "${PDC_PACKAGE}") if(NOT MPI_RUN_CMD) set(MPI_RUN_CMD mpiexec) endif() + +#------------------------------------------------------------------------------ +# general cmake flags: +# -DCMAKE_INSTALL_PREFIX=/usr/local -- the prefix for installing +# -DCMAKE_BUILD_TYPE=type -- type can be Debug, Release, ... +# -DCMAKE_PREFIX_PATH=/dir -- external packages +# +# note that CMAKE_PREFIX_PATH can be a list of directories: +# -DCMAKE_PREFIX_PATH='/dir1;/dir2;/dir3' +#------------------------------------------------------------------------------ + + #------------------------------------------------------------------------------ # Setup install and output Directories #------------------------------------------------------------------------------ @@ -79,6 +91,19 @@ if(NOT CMAKE_INSTALL_RPATH_USE_LINK_PATH) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) endif() +#------------------------------------------------------------------------------ +# Setup CMake Prefix Paths for searching external libraries. +#------------------------------------------------------------------------------ +# note that CMAKE_PREFIX_PATH can be a list of directories: +# -DCMAKE_PREFIX_PATH='/dir1;/dir2;/dir3' + +if(NOT CMAKE_PREFIX_PATH) + set(CMAKE_PREFIX_PATH ${CMAKE_INSTALL_PREFIX}) +endif() +# MERCURY +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} $ENV{MERCURY_DIR}) +# ANY Future external package goes here... + #------------------------------------------------------------------------------ # Setup CMake Environment #------------------------------------------------------------------------------ From 398c0b900e3e25be94357ae075d9628cc48635e1 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 10 Jul 2023 15:55:17 -0400 Subject: [PATCH 204/216] update commons/CMakeLists.txt --- src/commons/CMakeLists.txt | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/commons/CMakeLists.txt b/src/commons/CMakeLists.txt index 485ec4b8a..d0f83bb54 100644 --- a/src/commons/CMakeLists.txt +++ b/src/commons/CMakeLists.txt @@ -19,19 +19,12 @@ endif() # Mercury -find_package(MERCURY REQUIRED) +find_package(MERCURY CONFIG REQUIRED) if(MERCURY_FOUND) - message(STATUS "mercury dir = ${MERCURY_DIR}") - # NOTE: enable the following if you need ${MERCURY_INCLUDE_DIR} in the future - # NOTE: remember to add MERCURY_HOME to PATH or CMAKE_PREFIX_PATH if you enable the following. - find_path(MERCURY_INCLUDE_DIR mercury.h HINTS ${MERCURY_DIR}) - find_library(MERCURY_LIBRARY mercury HINTS ${MERCURY_DIR}) - message(STATUS "mercury include dir = ${MERCURY_INCLUDE_DIR}") - message(STATUS "mercury lib = ${MERCURY_LIBRARY}") - set(PDC_EXT_INCLUDE_DEPENDENCIES ${MERCURY_INCLUDE_DIR} + set(PDC_EXT_INCLUDE_DEPENDENCIES ${MERCURY_INCLUDE_DIRS} ${PDC_EXT_INCLUDE_DEPENDENCIES} ) - set(PDC_EXT_LIB_DEPENDENCIES mercury ${PDC_EXT_LIB_DEPENDENCIES}) + set(PDC_EXT_LIB_DEPENDENCIES ${MERCURY_LIBRARIES} ${PDC_EXT_LIB_DEPENDENCIES}) endif() include_directories(${PDC_EXT_INCLUDE_DEPENDENCIES}) From 702d6058b6b2ab4f76e018ea8a754861a07d8ca7 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Tue, 18 Jul 2023 13:44:07 -0700 Subject: [PATCH 205/216] Fix unnecessary memory allocation (#103) --- src/api/pdc_obj/pdc_obj.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/pdc_obj/pdc_obj.c b/src/api/pdc_obj/pdc_obj.c index d402782f9..f16a71275 100644 --- a/src/api/pdc_obj/pdc_obj.c +++ b/src/api/pdc_obj/pdc_obj.c @@ -1237,9 +1237,9 @@ PDCobj_get_info(pdcid_t obj_id) tmp = PDC_obj_get_info(obj_id); - ret_value = PDC_CALLOC(struct pdc_obj_info); - if (!ret_value) - PGOTO_ERROR(NULL, "failed to allocate memory"); + /* ret_value = PDC_CALLOC(struct pdc_obj_info); */ + /* if (!ret_value) */ + /* PGOTO_ERROR(NULL, "failed to allocate memory"); */ ret_value = tmp->obj_info_pub; From 98dae84a5b580a22de4343e24ff46beca67682c9 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Wed, 19 Jul 2023 09:15:32 -0700 Subject: [PATCH 206/216] Fix an issue with opening a deleted container, added test (#101) * Fix an issue with opening a deleted container, added test * Refactor the query aggregation process --- src/api/pdc_client_connect.c | 21 +++++++++++---------- src/api/pdc_obj/pdc_cont.c | 3 +++ src/server/pdc_client_server_common.c | 5 ++++- src/tests/cont_del.c | 5 +++++ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index 467de4bca..e6ebe726f 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -2361,21 +2361,22 @@ PDC_Client_query_metadata_name_timestep_agg(const char *obj_name, int time_step, FUNC_ENTER(NULL); #ifdef ENABLE_MPI - if (pdc_client_mpi_rank_g == 0) { + if (pdc_client_mpi_rank_g == 0) ret_value = PDC_Client_query_metadata_name_timestep(obj_name, time_step, out, metadata_server_id); - if (ret_value != SUCCEED || NULL == *out) { - *out = (pdc_metadata_t *)calloc(1, sizeof(pdc_metadata_t)); - PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: - ERROR with query [%s]", pdc_client_mpi_rank_g, obj_name); - } - } - else + + MPI_Bcast(&ret_value, 1, MPI_INT, 0, PDC_CLIENT_COMM_WORLD_g); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: - ERROR with query [%s]", pdc_client_mpi_rank_g, obj_name); + + if (pdc_client_mpi_rank_g != 0) *out = (pdc_metadata_t *)calloc(1, sizeof(pdc_metadata_t)); MPI_Bcast(*out, sizeof(pdc_metadata_t), MPI_CHAR, 0, PDC_CLIENT_COMM_WORLD_g); - MPI_Bcast(metadata_server_id, 1, MPI_UINT32_T, 0, PDC_CLIENT_COMM_WORLD_g); #else ret_value = PDC_Client_query_metadata_name_timestep(obj_name, time_step, out, metadata_server_id); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: - ERROR with query [%s]", pdc_client_mpi_rank_g, obj_name); #endif done: @@ -5606,8 +5607,8 @@ PDC_Client_query_container_name_col(const char *cont_name, uint64_t *cont_meta_i MPI_Bcast(cont_meta_id, 1, MPI_LONG_LONG, 0, PDC_CLIENT_COMM_WORLD_g); #else - printf("==PDC_CLIENT[%d]: Calling MPI collective operation without enabling MPI!\n", - pdc_client_mpi_rank_g); + PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Calling MPI collective operation without enabling MPI!", + pdc_client_mpi_rank_g); #endif done: diff --git a/src/api/pdc_obj/pdc_cont.c b/src/api/pdc_obj/pdc_cont.c index 444596729..23f539390 100644 --- a/src/api/pdc_obj/pdc_cont.c +++ b/src/api/pdc_obj/pdc_cont.c @@ -274,6 +274,9 @@ PDCcont_open(const char *cont_name, pdcid_t pdc) ret = PDC_Client_query_container_name(cont_name, &cont_meta_id); if (ret == FAIL) PGOTO_ERROR(0, "query container name failed"); + if (cont_meta_id == 0) + PGOTO_ERROR(0, "query container not found"); + cont_id = PDC_cont_create_local(pdc, cont_name, cont_meta_id); ret_value = cont_id; diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index e3e8597b1..859c7234a 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -1685,7 +1685,10 @@ HG_TEST_RPC_CB(container_query, handle) HG_Get_input(handle, &in); PDC_Server_find_container_by_name(in.cont_name, &cont_entry); - out.cont_id = cont_entry->cont_id; + if (cont_entry) + out.cont_id = cont_entry->cont_id; + else + out.cont_id = 0; HG_Respond(handle, NULL, NULL, &out); diff --git a/src/tests/cont_del.c b/src/tests/cont_del.c index ab708685f..5ecdf2e6b 100644 --- a/src/tests/cont_del.c +++ b/src/tests/cont_del.c @@ -79,6 +79,11 @@ main(int argc, char **argv) printf("successfully close container c1\n"); } + printf("trying to open a deleted container, should fail\n"); + cont = PDCcont_open("VPIC_cont", pdc); + if (cont > 0) + printf("Error: opened a container that was just deleted @ line %d!\n", __LINE__); + // close a container property if (PDCprop_close(create_prop) < 0) { printf("Fail to close property @ line %d\n", __LINE__); From d1bdc1626aec2006bad1fa92a5ef703fd42a2d5c Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Wed, 19 Jul 2023 11:16:13 -0700 Subject: [PATCH 207/216] Fix container tag delete error (#102) * Fix container tag delete error * Committing clang-format changes * Update tag delete function * Refactor metdata lookup process for tag deletion * Committing clang-format changes * Formatting and comment * Committing clang-format changes --------- Co-authored-by: github-actions --- src/api/pdc_client_connect.c | 145 ++++++++++++++++------------- src/api/pdc_obj/include/pdc_prop.h | 10 -- src/server/pdc_server_metadata.c | 44 ++++----- src/tests/cont_tags.c | 71 ++++++++++---- src/tests/kvtag_add_get.c | 2 +- 5 files changed, 156 insertions(+), 116 deletions(-) diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index e6ebe726f..34cef9ad8 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -7058,16 +7058,20 @@ metadata_get_kvtag_rpc_cb(const struct hg_cb_info *callback_info) ret_value = HG_Get_output(handle, &output); if (ret_value != HG_SUCCESS) { client_lookup_args->ret = -1; - PGOTO_ERROR(ret_value, "==PDC_CLIENT[%d]: metadata_add_tag_rpc_cb error with HG_Get_output", - pdc_client_mpi_rank_g); + PGOTO_ERROR(ret_value, "==PDC_CLIENT[%d]: %s error with HG_Get_output", pdc_client_mpi_rank_g, + __func__); + } + client_lookup_args->ret = output.ret; + if (output.kvtag.name) + client_lookup_args->kvtag->name = strdup(output.kvtag.name); + client_lookup_args->kvtag->size = output.kvtag.size; + client_lookup_args->kvtag->type = output.kvtag.type; + if (output.kvtag.size > 0) { + client_lookup_args->kvtag->value = malloc(output.kvtag.size); + memcpy(client_lookup_args->kvtag->value, output.kvtag.value, output.kvtag.size); } - client_lookup_args->ret = output.ret; - client_lookup_args->kvtag->name = strdup(output.kvtag.name); - client_lookup_args->kvtag->size = output.kvtag.size; - client_lookup_args->kvtag->type = output.kvtag.type; - client_lookup_args->kvtag->value = malloc(output.kvtag.size); - memcpy(client_lookup_args->kvtag->value, output.kvtag.value, output.kvtag.size); - /* PDC_kvtag_dup(&(output.kvtag), &client_lookup_args->kvtag); */ + else + client_lookup_args->kvtag->value = NULL; done: fflush(stdout); @@ -7118,13 +7122,13 @@ PDC_get_kvtag(pdcid_t obj_id, char *tag_name, pdc_kvtag_t **kvtag, int is_cont) in.key = tag_name; } else - PGOTO_ERROR(FAIL, "==PDC_Client_get_kvtag(): invalid tag content!"); + PGOTO_ERROR(FAIL, "PDC_get_kvtag: invalid tag content!"); *kvtag = (pdc_kvtag_t *)malloc(sizeof(pdc_kvtag_t)); lookup_args.kvtag = *kvtag; hg_ret = HG_Forward(metadata_get_kvtag_handle, metadata_get_kvtag_rpc_cb, &lookup_args, &in); if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(FAIL, "PDC_Client_get_kvtag_metadata_with_name(): Could not start HG_Forward()"); + PGOTO_ERROR(FAIL, "PDC_get_kvtag: Could not start HG_Forward()"); // Wait for response from server work_todo_g = 1; @@ -7140,55 +7144,6 @@ PDC_get_kvtag(pdcid_t obj_id, char *tag_name, pdc_kvtag_t **kvtag, int is_cont) FUNC_LEAVE(ret_value); } -perr_t -PDCtag_delete(pdcid_t obj_id, char *tag_name) -{ - perr_t ret_value = SUCCEED; - hg_return_t hg_ret = 0; - uint64_t meta_id; - uint32_t server_id; - hg_handle_t metadata_del_kvtag_handle; - metadata_get_kvtag_in_t in; - struct _pdc_obj_info * obj_prop; - struct _pdc_client_lookup_args lookup_args; - - FUNC_ENTER(NULL); - - obj_prop = PDC_obj_get_info(obj_id); - meta_id = obj_prop->obj_info_pub->meta_id; - server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); - - debug_server_id_count[server_id]++; - - if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) - PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); - - HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_del_kvtag_register_id_g, - &metadata_del_kvtag_handle); - - // Fill input structure - in.obj_id = meta_id; - in.hash_value = PDC_get_hash_by_name(obj_prop->obj_info_pub->name); - in.key = tag_name; - - hg_ret = HG_Forward(metadata_del_kvtag_handle, metadata_add_tag_rpc_cb /*reuse*/, &lookup_args, &in); - if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(FAIL, "PDC_Client_del_kvtag_metadata_with_name(): Could not start HG_Forward()"); - - // Wait for response from server - work_todo_g = 1; - PDC_Client_check_response(&send_context_g); - - if (lookup_args.ret != 1) - printf("PDC_CLIENT: del kvtag NOT successful ... ret_value = %d\n", lookup_args.ret); - -done: - fflush(stdout); - HG_Destroy(metadata_del_kvtag_handle); - - FUNC_LEAVE(ret_value); -} - static hg_return_t kvtag_query_bulk_cb(const struct hg_cb_info *hg_cb_info) { @@ -7455,6 +7410,69 @@ PDC_Client_query_kvtag_col(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ FUNC_LEAVE(ret_value); } +// Delete a tag specified by a name, and whether it is from a container or an object +static perr_t +PDCtag_delete(pdcid_t obj_id, char *tag_name, int is_cont) +{ + perr_t ret_value = SUCCEED; + hg_return_t hg_ret = 0; + uint64_t meta_id; + uint32_t server_id; + hg_handle_t metadata_del_kvtag_handle; + metadata_get_kvtag_in_t in; + struct _pdc_obj_info * obj_prop; + struct _pdc_cont_info * cont_prop; + struct _pdc_client_lookup_args lookup_args; + + FUNC_ENTER(NULL); + + if (is_cont) { + cont_prop = PDC_cont_get_info(obj_id); + meta_id = cont_prop->cont_info_pub->meta_id; + } + else { + obj_prop = PDC_obj_get_info(obj_id); + meta_id = obj_prop->obj_info_pub->meta_id; + } + + server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); + + debug_server_id_count[server_id]++; + + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) + PGOTO_ERROR(FAIL, "==CLIENT[%d]: ERROR with PDC_Client_try_lookup_server", pdc_client_mpi_rank_g); + + HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_del_kvtag_register_id_g, + &metadata_del_kvtag_handle); + + // Fill input structure + in.obj_id = meta_id; + + if (is_cont) + in.hash_value = PDC_get_hash_by_name(cont_prop->cont_info_pub->name); + else + in.hash_value = PDC_get_hash_by_name(obj_prop->obj_info_pub->name); + in.key = tag_name; + + // reuse metadata_add_tag_rpc_cb here since it only checks the return value + hg_ret = HG_Forward(metadata_del_kvtag_handle, metadata_add_tag_rpc_cb /*reuse*/, &lookup_args, &in); + if (hg_ret != HG_SUCCESS) + PGOTO_ERROR(FAIL, "PDC_Client_del_kvtag_metadata_with_name(): Could not start HG_Forward()"); + + // Wait for response from server + work_todo_g = 1; + PDC_Client_check_response(&send_context_g); + + if (lookup_args.ret != 1) + printf("PDC_CLIENT: del kvtag NOT successful ... ret_value = %d\n", lookup_args.ret); + +done: + fflush(stdout); + HG_Destroy(metadata_del_kvtag_handle); + + FUNC_LEAVE(ret_value); +} + /* - -------------------------------- */ /* New Simple Object Access Interface */ /* - -------------------------------- */ @@ -7484,7 +7502,6 @@ PDCcont_put(const char *cont_name, pdcid_t pdc) } pdcid_t - PDCcont_get_id(const char *cont_name, pdcid_t pdc_id) { pdcid_t cont_id; @@ -7634,9 +7651,9 @@ PDCcont_del_tag(pdcid_t cont_id, char *tag_name) FUNC_ENTER(NULL); - ret_value = PDCobj_del_tag(cont_id, tag_name); + ret_value = PDCtag_delete(cont_id, tag_name, 1); if (ret_value != SUCCEED) - PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error with PDCobj_del_tag", pdc_client_mpi_rank_g); + PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error with PDCtag_delete", pdc_client_mpi_rank_g); done: fflush(stdout); @@ -7832,7 +7849,7 @@ PDCobj_del_tag(pdcid_t obj_id, char *tag_name) FUNC_ENTER(NULL); - ret_value = PDCtag_delete(obj_id, tag_name); + ret_value = PDCtag_delete(obj_id, tag_name, 0); if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: Error with PDC_del_kvtag", pdc_client_mpi_rank_g); diff --git a/src/api/pdc_obj/include/pdc_prop.h b/src/api/pdc_obj/include/pdc_prop.h index 37ff599eb..0459e0283 100644 --- a/src/api/pdc_obj/include/pdc_prop.h +++ b/src/api/pdc_obj/include/pdc_prop.h @@ -108,16 +108,6 @@ struct pdc_obj_prop *PDCobj_prop_get_info(pdcid_t prop_id); */ perr_t PDCprop_update(pdcid_t obj_id, pdcid_t prop_id); -/** - * Delete a tag with a specific name and value - * - * \param obj_id[IN] Object ID - * \param tag_name [IN] Metadta field name - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDCtag_delete(pdcid_t obj_id, char *tag_name); - /** * ********** * diff --git a/src/server/pdc_server_metadata.c b/src/server/pdc_server_metadata.c index 2331df3b1..6997adf8c 100644 --- a/src/server/pdc_server_metadata.c +++ b/src/server/pdc_server_metadata.c @@ -2778,7 +2778,8 @@ PDC_Server_del_kvtag(metadata_get_kvtag_in_t *in, metadata_add_tag_out_t *out) #ifdef ENABLE_MULTITHREAD int unlocked; #endif - pdc_hash_table_entry_head *lookup_value; + pdc_hash_table_entry_head * lookup_value; + pdc_cont_hash_table_entry_t *cont_lookup_value; FUNC_ENTER(NULL); @@ -2794,62 +2795,63 @@ PDC_Server_del_kvtag(metadata_get_kvtag_in_t *in, metadata_add_tag_out_t *out) #ifdef ENABLE_MULTITHREAD // Obtain lock for hash table - unlocked = 0; hg_thread_mutex_lock(&pdc_metadata_hash_table_mutex_g); #endif + // Look obj tags first lookup_value = hash_table_lookup(metadata_hash_table_g, &hash_key); if (lookup_value != NULL) { pdc_metadata_t *target; target = find_metadata_by_id_from_list(lookup_value->metadata, obj_id); if (target != NULL) { - PDC_del_kvtag_value_from_list(&target->kvtag_list_head, in->key); - out->ret = 1; + ret_value = PDC_del_kvtag_value_from_list(&target->kvtag_list_head, in->key); + out->ret = 1; } else { ret_value = FAIL; out->ret = -1; + printf("==PDC_SERVER[%d]: %s - failed to find requested kvtag [%s]\n", pdc_server_rank_g, + __func__, in->key); + goto done; } } else { - ret_value = FAIL; - out->ret = -1; - } - - if (ret_value != SUCCEED) { - printf("==PDC_SERVER[%d]: %s - error \n", pdc_server_rank_g, __func__); - goto done; + cont_lookup_value = hash_table_lookup(container_hash_table_g, &hash_key); + if (cont_lookup_value != NULL) { + PDC_del_kvtag_value_from_list(&cont_lookup_value->kvtag_list_head, in->key); + out->ret = 1; + } + else { + ret_value = FAIL; + out->ret = -1; + printf("==PDC_SERVER[%d]: %s - failed to find requested kvtag [%s]\n", pdc_server_rank_g, + __func__, in->key); + goto done; + } } +done: #ifdef ENABLE_MULTITHREAD - // ^ Release hash table lock hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); - unlocked = 1; #endif #ifdef ENABLE_TIMING // Timing gettimeofday(&pdc_timer_end, 0); ht_total_sec = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); -#endif #ifdef ENABLE_MULTITHREAD hg_thread_mutex_lock(&pdc_time_mutex_g); #endif -#ifdef ENABLE_TIMING server_update_time_g += ht_total_sec; -#endif #ifdef ENABLE_MULTITHREAD hg_thread_mutex_unlock(&pdc_time_mutex_g); #endif -done: -#ifdef ENABLE_MULTITHREAD - if (unlocked == 0) - hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); -#endif +#endif // End ENABLE_TIMING + fflush(stdout); FUNC_LEAVE(ret_value); diff --git a/src/tests/cont_tags.c b/src/tests/cont_tags.c index caaf87b49..2f801acc4 100644 --- a/src/tests/cont_tags.c +++ b/src/tests/cont_tags.c @@ -32,7 +32,6 @@ main(int argc, char **argv) { pdcid_t pdc, cont_prop, cont, cont2; perr_t ret; - int ret_value = 0; int rank = 0, size = 1; @@ -58,7 +57,7 @@ main(int argc, char **argv) } else { printf("Fail to create container property @ line %d!\n", __LINE__); - ret_value = 1; + return -1; } // create a container cont = PDCcont_create("c1", cont_prop); @@ -67,7 +66,7 @@ main(int argc, char **argv) } else { printf("Fail to create container @ line %d!\n", __LINE__); - ret_value = 1; + return -1; } cont2 = PDCcont_create("c2", cont_prop); @@ -76,88 +75,120 @@ main(int argc, char **argv) } else { printf("Fail to create container @ line %d!\n", __LINE__); - ret_value = 1; + return -1; } ret = PDCcont_put_tag(cont, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 1\n"); - ret_value = 1; + return -1; } ret = PDCcont_put_tag(cont, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 1\n"); - ret_value = 1; + return -1; } ret = PDCcont_put_tag(cont2, "some tag", tag_value, PDC_STRING, strlen(tag_value) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 2\n"); - ret_value = 1; + return -1; } ret = PDCcont_put_tag(cont2, "some tag 2", tag_value2, PDC_STRING, strlen(tag_value2) + 1); if (ret != SUCCEED) { printf("Put tag failed at container 2\n"); - ret_value = 1; + return -1; } ret = PDCcont_get_tag(cont, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 1\n"); - ret_value = 1; + return -1; } if (strcmp(tag_value, tag_value_ret) != 0) { printf("Wrong tag value at container 1, expected = [%s], get [%s]\n", tag_value, tag_value_ret); - ret_value = 1; + return -1; } ret = PDCcont_get_tag(cont, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 1\n"); - ret_value = 1; + return -1; } if (strcmp(tag_value2, tag_value_ret) != 0) { printf("Wrong tag value at container 1, expected = [%s], get [%s]\n", tag_value2, tag_value_ret); - ret_value = 1; + return -1; } ret = PDCcont_get_tag(cont2, "some tag", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 2\n"); - ret_value = 1; + return -1; } if (strcmp(tag_value, tag_value_ret) != 0) { printf("Wrong tag value at container 2, expected = [%s], get [%s]\n", tag_value, tag_value_ret); - ret_value = 1; + return -1; } ret = PDCcont_get_tag(cont2, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); if (ret != SUCCEED) { printf("Get tag failed at container 2\n"); - ret_value = 1; + return -1; } if (strcmp(tag_value2, tag_value_ret) != 0) { printf("Wrong tag value at container 2, expected = [%s], get [%s]\n", tag_value2, tag_value_ret); - ret_value = 1; + return -1; + } + + ret = PDCcont_del_tag(cont2, "some tag 2"); + if (ret != SUCCEED) { + printf("Delete tag failed at container 2\n"); + return -1; + } + else { + printf("successfully deleted a tag from container c2\n"); + } + + ret = PDCcont_get_tag(cont2, "some tag 2", (void **)&tag_value_ret, &value_type, &value_size); + if (ret != SUCCEED) { + printf("Get tag failed at container 2\n"); + return -1; + } + + if (tag_value_ret != NULL || value_size != 0) { + printf("Error: got non-empty tag after deletion\n"); + return -1; + } + else { + printf("verified the tag has been deleted successfully\n"); } // close a container if (PDCcont_close(cont) < 0) { printf("fail to close container c1\n"); - ret_value = 1; + return -1; + } + else { + printf("successfully close container c1\n"); + } + // close a container + if (PDCcont_close(cont2) < 0) { + printf("fail to close container c1\n"); + return -1; } else { printf("successfully close container c1\n"); } + // close a container property if (PDCprop_close(cont_prop) < 0) { printf("Fail to close property @ line %d\n", __LINE__); - ret_value = 1; + return -1; } else { printf("successfully close container property\n"); @@ -165,10 +196,10 @@ main(int argc, char **argv) // close pdc if (PDCclose(pdc) < 0) { printf("fail to close PDC\n"); - ret_value = 1; + return -1; } #ifdef ENABLE_MPI MPI_Finalize(); #endif - return ret_value; + return 0; } diff --git a/src/tests/kvtag_add_get.c b/src/tests/kvtag_add_get.c index 7d53da6a6..ad06b506b 100644 --- a/src/tests/kvtag_add_get.c +++ b/src/tests/kvtag_add_get.c @@ -132,7 +132,7 @@ main() else printf("successfully retrieved a kvtag [%s] = [%f] from o2\n", kvtag3.name, *(double *)value3); - if (PDCtag_delete(obj1, kvtag1.name) < 0) + if (PDCobj_del_tag(obj1, kvtag1.name) < 0) printf("fail to delete a kvtag from o1\n"); else printf("successfully deleted a kvtag [%s] from o1\n", kvtag1.name); From 55a5628eb009bee71171978df8dae2d140f2fd21 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Thu, 27 Jul 2023 10:49:22 -0700 Subject: [PATCH 208/216] Fix the server cache issue when cache becomes full and needs flush (#113) * Fix the server cache issue when cache becomes full and need flush * Committing clang-format changes --------- Co-authored-by: github-actions --- .../pdc_server_region_cache.c | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/server/pdc_server_region/pdc_server_region_cache.c b/src/server/pdc_server_region/pdc_server_region_cache.c index 92c408fa6..d2f2abe56 100644 --- a/src/server/pdc_server_region/pdc_server_region_cache.c +++ b/src/server/pdc_server_region/pdc_server_region_cache.c @@ -393,8 +393,11 @@ PDC_region_cache_register(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dim struct pdc_region_info *region_cache_info; if (obj_ndim != ndim && obj_ndim > 0) { printf("PDC_region_cache_register reports obj_ndim != ndim, %d != %d\n", obj_ndim, ndim); + return FAIL; } + pthread_mutex_lock(&pdc_obj_cache_list_mutex); + obj_cache_iter = obj_cache_list; while (obj_cache_iter != NULL) { if (obj_cache_iter->obj_id == obj_id) { @@ -460,7 +463,15 @@ PDC_region_cache_register(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dim memcpy(region_cache_info->buf, buf, sizeof(char) * buf_size); total_cache_size += buf_size; + pthread_mutex_unlock(&pdc_obj_cache_list_mutex); + if (total_cache_size > maximum_cache_size) { + int server_rank = 0; +#ifdef ENABLE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &server_rank); +#endif + printf("==PDC_SERVER[%d]: server cache full %.1f / %.1f MB, will flush to storage\n", server_rank, + total_cache_size / 1048576.0, maximum_cache_size / 1048576.0); PDC_region_cache_flush_all(); } @@ -571,11 +582,11 @@ PDC_transfer_request_data_write_out(uint64_t obj_id, int obj_ndim, const uint64_ region_cache_iter = region_cache_iter->next; } } + pthread_mutex_unlock(&pdc_obj_cache_list_mutex); if (!flag) { PDC_region_cache_register(obj_id, obj_ndim, obj_dims, buf, write_size, region_info->offset, region_info->size, region_info->ndim, unit); } - pthread_mutex_unlock(&pdc_obj_cache_list_mutex); // PDC_Server_data_write_out2(obj_id, region_info, buf, unit); #ifdef PDC_TIMING @@ -862,10 +873,9 @@ PDC_region_cache_clock_cycle(void *ptr) gettimeofday(&finish_time, NULL); elapsed_time = finish_time.tv_sec - current_time.tv_sec + (finish_time.tv_usec - current_time.tv_usec) / 1000000.0; - fprintf( - stderr, - "==PDC_SERVER[%d]: flushed %d regions from cache to storage (every %.1fs), took %.4fs\n", - server_rank, nflush, flush_frequency_s, elapsed_time); + fprintf(stderr, + "==PDC_SERVER[%d]: flushed %d regions to storage (full/every %.0fs), took %.4fs\n", + server_rank, nflush, flush_frequency_s, elapsed_time); } pthread_mutex_unlock(&pdc_obj_cache_list_mutex); } From 0cc87e7acc13d9584c47bde601e1e6d49d0664d8 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Thu, 27 Jul 2023 13:50:41 -0400 Subject: [PATCH 209/216] Fix a wrong option description. (#115) * upate metadata type system * update serde framework to coupe with the new data type system * replace unnecessary data types * adding type for pdc_kvtag_t, all occurances are fixed * update new commons CMake for publishing commons * commons compilation passed * compiled * remove unnecessary header files from installation * resolve conflict * add important files * clang formatting * update cmake * update option message --------- Co-authored-by: Houjun Tang --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2deac8837..715766747 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -336,7 +336,7 @@ endif() #----------------------------------------------------------------------------- # SERVER CACHE option #----------------------------------------------------------------------------- -option(PDC_SERVER_CACHE "Enable timing." OFF) +option(PDC_SERVER_CACHE "Enable Server Caching." OFF) if(PDC_SERVER_CACHE) set(PDC_SERVER_CACHE 1) endif() From bc245e1b3869d11fa3f30fcc4862e466803aa33a Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Mon, 31 Jul 2023 10:19:26 -0700 Subject: [PATCH 210/216] Install header files needed by PDCpy (#114) --- src/api/CMakeLists.txt | 5 ++++- src/api/pdc_obj/pdc_obj.c | 6 ++---- src/commons/CMakeLists.txt | 2 ++ src/commons/utils/include/pdc_id_pkg.h | 5 +++++ src/commons/utils/include/pdc_private.h | 5 ----- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index cfbb8ac21..8a6b6492e 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -157,6 +157,9 @@ set(PDC_HEADERS ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_obj.h ${PDC_SOURCE_DIR}/src/api/pdc_region/include/pdc_region.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_prop.h + ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_cont_pkg.h + ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_obj_pkg.h + ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_prop_pkg.h ${PDC_SOURCE_DIR}/src/api/pdc_query/include/pdc_query.h ${PDC_SOURCE_DIR}/src/api/pdc_region/include/pdc_region.h ${PDC_SOURCE_DIR}/src/api/pdc_transform/include/pdc_transform.h @@ -239,4 +242,4 @@ set(PDC_INCLUDES_INSTALL_TIME ${PDC_INSTALL_INCLUDE_DIR} ${PDC_EXT_INCLUDE_DEPENDENCIES} PARENT_SCOPE -) \ No newline at end of file +) diff --git a/src/api/pdc_obj/pdc_obj.c b/src/api/pdc_obj/pdc_obj.c index f16a71275..83325ba29 100644 --- a/src/api/pdc_obj/pdc_obj.c +++ b/src/api/pdc_obj/pdc_obj.c @@ -1236,10 +1236,8 @@ PDCobj_get_info(pdcid_t obj_id) /* obj_id = PDC_find_byname(PDC_OBJ, obj_name); */ tmp = PDC_obj_get_info(obj_id); - - /* ret_value = PDC_CALLOC(struct pdc_obj_info); */ - /* if (!ret_value) */ - /* PGOTO_ERROR(NULL, "failed to allocate memory"); */ + if (NULL == tmp) + PGOTO_ERROR(NULL, "failed to allocate memory"); ret_value = tmp->obj_info_pub; diff --git a/src/commons/CMakeLists.txt b/src/commons/CMakeLists.txt index d0f83bb54..04fcd7056 100644 --- a/src/commons/CMakeLists.txt +++ b/src/commons/CMakeLists.txt @@ -59,6 +59,8 @@ message(STATUS "PDC_COMMONS_INCLUDE_DIRS: ${PDC_COMMONS_INCLUDE_DIRS}") install( FILES ${CMAKE_BINARY_DIR}/pdc_config.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/pdc_id_pkg.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/pdc_malloc.h DESTINATION ${PDC_INSTALL_INCLUDE_DIR} COMPONENT diff --git a/src/commons/utils/include/pdc_id_pkg.h b/src/commons/utils/include/pdc_id_pkg.h index 9623661fd..c0ac3afce 100644 --- a/src/commons/utils/include/pdc_id_pkg.h +++ b/src/commons/utils/include/pdc_id_pkg.h @@ -47,6 +47,11 @@ /* Map an atom to an ID type number */ #define PDC_TYPE(a) ((PDC_type_t)(((pdcid_t)(a) >> ID_BITS) & TYPE_MASK)) +struct _pdc_class { + char * name; + pdcid_t local_id; +}; + struct _pdc_id_info { pdcid_t id; /* ID for this info */ hg_atomic_int32_t count; /* ref. count for this atom */ diff --git a/src/commons/utils/include/pdc_private.h b/src/commons/utils/include/pdc_private.h index b0fa48f14..3fee1baaa 100644 --- a/src/commons/utils/include/pdc_private.h +++ b/src/commons/utils/include/pdc_private.h @@ -67,11 +67,6 @@ typedef enum { C_lang = 0, FORTRAN_lang, PYTHON_lang, JULIA_lang, N_LANGUAGES } /***************************/ /* Library Private Structs */ /***************************/ -struct _pdc_class { - char * name; - pdcid_t local_id; -}; - #ifdef __cplusplus #define ATTRIBUTE(a) #else /* __cplusplus */ From e9abc584a2b29e9351410f79ada0bc39698b8b4d Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Wed, 2 Aug 2023 14:37:45 -0700 Subject: [PATCH 211/216] Support unlimited object dimension size (#117) * Support unlimited object dimension szie * Add function description for PDC_SIZE_UNLIMITED --- src/api/pdc_obj/include/pdc_obj.h | 2 +- src/commons/include/pdc_public.h | 4 +++- src/tests/region_transfer.c | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/api/pdc_obj/include/pdc_obj.h b/src/api/pdc_obj/include/pdc_obj.h index 8ad7a285a..4ce3d6bb5 100644 --- a/src/api/pdc_obj/include/pdc_obj.h +++ b/src/api/pdc_obj/include/pdc_obj.h @@ -228,7 +228,7 @@ perr_t PDCprop_set_obj_tags(pdcid_t obj_prop, char *tags); * * \param obj_prop [IN] ID of object property, returned by PDCprop_create(PDC_OBJ_CREATE) * \param ndim [IN] Number of dimensions - * \param dims [IN] Size of each dimension + * \param dims [IN] Size of each dimension, positive value, can be PDC_SIZE_UNLIMITED * * \return Non-negative on success/Negative on failure */ diff --git a/src/commons/include/pdc_public.h b/src/commons/include/pdc_public.h index 6c135f657..cd7496723 100644 --- a/src/commons/include/pdc_public.h +++ b/src/commons/include/pdc_public.h @@ -48,7 +48,7 @@ typedef enum { PDC_PERSIST, PDC_TRANSIENT } pdc_lifetime_t; typedef enum { PDC_SERVER_DEFAULT = 0, PDC_SERVER_PER_CLIENT = 1 } pdc_server_selection_t; -typedef struct pdc_histogram_t { //????????? +typedef struct pdc_histogram_t { pdc_var_type_t dtype; int nbin; double incr; @@ -59,4 +59,6 @@ typedef struct pdc_histogram_t { //????????? #define SUCCEED 0 #define FAIL (-1) +#define PDC_SIZE_UNLIMITED UINT64_MAX + #endif /* PDC_PUBLIC_H */ diff --git a/src/tests/region_transfer.c b/src/tests/region_transfer.c index 67a2880f2..3a3b04547 100644 --- a/src/tests/region_transfer.c +++ b/src/tests/region_transfer.c @@ -57,7 +57,7 @@ main(int argc, char **argv) int *data = (int *)malloc(sizeof(int) * BUF_LEN); int *data_read = (int *)malloc(sizeof(int) * BUF_LEN); - dims[0] = BUF_LEN; + dims[0] = PDC_SIZE_UNLIMITED; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); From 38a34fe9ba074ace1d279f9326338715396d20aa Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Wed, 2 Aug 2023 14:46:57 -0700 Subject: [PATCH 212/216] Fix obj_round_robin_io_all test code --- src/tests/obj_round_robin_io_all.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tests/obj_round_robin_io_all.c b/src/tests/obj_round_robin_io_all.c index 05a5c0dcc..500779213 100644 --- a/src/tests/obj_round_robin_io_all.c +++ b/src/tests/obj_round_robin_io_all.c @@ -108,9 +108,9 @@ main(int argc, char **argv) my_data_size *= dims[i]; } - mydata = (char **)malloc(size * WRITE_REQ_SIZE); + mydata = (char **)malloc(2 * sizeof(char*)); mydata[0] = (char *)malloc(my_data_size * type_size); - mydata[1] = mydata[0] + my_data_size * type_size; + mydata[1] = (char *)malloc(my_data_size * type_size); offset = (uint64_t *)malloc(sizeof(uint64_t) * ndim); mysize = (uint64_t *)malloc(sizeof(uint64_t)); @@ -453,6 +453,8 @@ main(int argc, char **argv) free(obj1_list); free(obj2_list); free(data_read); + free(mydata[0]); + free(mydata[1]); free(mydata); free(offset); free(mysize); From 5e76eda840c0b36fe68b84d379428fea2ecefe7a Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 2 Aug 2023 21:47:36 +0000 Subject: [PATCH 213/216] Committing clang-format changes --- src/tests/obj_round_robin_io_all.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/obj_round_robin_io_all.c b/src/tests/obj_round_robin_io_all.c index 500779213..4070107a7 100644 --- a/src/tests/obj_round_robin_io_all.c +++ b/src/tests/obj_round_robin_io_all.c @@ -108,7 +108,7 @@ main(int argc, char **argv) my_data_size *= dims[i]; } - mydata = (char **)malloc(2 * sizeof(char*)); + mydata = (char **)malloc(2 * sizeof(char *)); mydata[0] = (char *)malloc(my_data_size * type_size); mydata[1] = (char *)malloc(my_data_size * type_size); From cf21137a2fc6da7d11d3e6f96467cbbd88ccd87d Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Thu, 3 Aug 2023 11:28:58 -0700 Subject: [PATCH 214/216] More header files fix for PDCpy --- .../include/pdc_analysis_and_transforms_common.h | 3 ++- src/api/pdc_analysis/include/pdc_hist_pkg.h | 1 + src/api/pdc_analysis/pdc_hist_pkg.c | 1 - src/api/pdc_obj/include/pdc_obj_pkg.h | 3 +-- src/api/pdc_obj/include/pdc_prop_pkg.h | 5 +++-- src/api/pdc_transform/include/pdc_transforms_pkg.h | 2 ++ src/commons/CMakeLists.txt | 1 + src/commons/utils/include/pdc_id_pkg.h | 2 +- src/commons/utils/include/pdc_private.h | 4 ---- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h b/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h index 082a3dd02..79ccbf3db 100644 --- a/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h +++ b/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h @@ -24,7 +24,8 @@ #ifndef PDC_OBJ_ANALYSIS_H #define PDC_OBJ_ANALYSIS_H -#include "pdc_private.h" +#include "pdc_prop_pkg.h" +#include "pdc_transforms_pkg.h" #include "mercury_proc_string.h" #include "mercury_atomic.h" #include diff --git a/src/api/pdc_analysis/include/pdc_hist_pkg.h b/src/api/pdc_analysis/include/pdc_hist_pkg.h index 3697743c3..6be68af7f 100644 --- a/src/api/pdc_analysis/include/pdc_hist_pkg.h +++ b/src/api/pdc_analysis/include/pdc_hist_pkg.h @@ -25,6 +25,7 @@ #ifndef PDC_HIST_H #define PDC_HIST_H +#include "pdc_private.h" #include "pdc_public.h" #include "math.h" #include diff --git a/src/api/pdc_analysis/pdc_hist_pkg.c b/src/api/pdc_analysis/pdc_hist_pkg.c index 47c93a94d..75343a144 100644 --- a/src/api/pdc_analysis/pdc_hist_pkg.c +++ b/src/api/pdc_analysis/pdc_hist_pkg.c @@ -1,5 +1,4 @@ #include "pdc_hist_pkg.h" -#include "pdc_private.h" #include #include diff --git a/src/api/pdc_obj/include/pdc_obj_pkg.h b/src/api/pdc_obj/include/pdc_obj_pkg.h index f08343a70..11effda8e 100644 --- a/src/api/pdc_obj/include/pdc_obj_pkg.h +++ b/src/api/pdc_obj/include/pdc_obj_pkg.h @@ -24,8 +24,7 @@ #ifndef PDC_OBJ_PKG_H #define PDC_OBJ_PKG_H - -#include "pdc_private.h" +#include "pdc_public.h" /****************************/ /* Library Private Typedefs */ diff --git a/src/api/pdc_obj/include/pdc_prop_pkg.h b/src/api/pdc_obj/include/pdc_prop_pkg.h index 52d80efa1..a97f1bfda 100644 --- a/src/api/pdc_obj/include/pdc_prop_pkg.h +++ b/src/api/pdc_obj/include/pdc_prop_pkg.h @@ -24,8 +24,7 @@ #ifndef PDC_PROP_PKG_H #define PDC_PROP_PKG_H - -#include "pdc_private.h" +#include "pdc_public.h" /*******************/ /* Private Typedefs */ @@ -43,6 +42,8 @@ typedef struct pdc_kvtag_t { void * value; } pdc_kvtag_t; +typedef enum { ROW_major, COL_major } _pdc_major_type_t; + struct _pdc_transform_state { _pdc_major_type_t storage_order; pdc_var_type_t dtype; diff --git a/src/api/pdc_transform/include/pdc_transforms_pkg.h b/src/api/pdc_transform/include/pdc_transforms_pkg.h index 3b4153152..4929f1f12 100644 --- a/src/api/pdc_transform/include/pdc_transforms_pkg.h +++ b/src/api/pdc_transform/include/pdc_transforms_pkg.h @@ -32,6 +32,8 @@ /***************************/ /* Library Private Structs */ /***************************/ +typedef enum { C_lang = 0, FORTRAN_lang, PYTHON_lang, JULIA_lang, N_LANGUAGES } _pdc_analysis_language_t; + struct _pdc_region_transform_ftn_info { pdcid_t object_id; pdcid_t region_id; diff --git a/src/commons/CMakeLists.txt b/src/commons/CMakeLists.txt index 04fcd7056..ab706bcbe 100644 --- a/src/commons/CMakeLists.txt +++ b/src/commons/CMakeLists.txt @@ -61,6 +61,7 @@ install( ${CMAKE_BINARY_DIR}/pdc_config.h ${PDC_SOURCE_DIR}/src/commons/utils/include/pdc_id_pkg.h ${PDC_SOURCE_DIR}/src/commons/utils/include/pdc_malloc.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/pdc_linkedlist.h DESTINATION ${PDC_INSTALL_INCLUDE_DIR} COMPONENT diff --git a/src/commons/utils/include/pdc_id_pkg.h b/src/commons/utils/include/pdc_id_pkg.h index c0ac3afce..ba858c82d 100644 --- a/src/commons/utils/include/pdc_id_pkg.h +++ b/src/commons/utils/include/pdc_id_pkg.h @@ -25,7 +25,7 @@ #ifndef PDC_ID_PKG_H #define PDC_ID_PKG_H -#include "pdc_private.h" +#include "pdc_public.h" #include "pdc_linkedlist.h" #include "mercury_atomic.h" /* diff --git a/src/commons/utils/include/pdc_private.h b/src/commons/utils/include/pdc_private.h index 3fee1baaa..e47911fd9 100644 --- a/src/commons/utils/include/pdc_private.h +++ b/src/commons/utils/include/pdc_private.h @@ -60,10 +60,6 @@ typedef enum { PDC_Q_MATCH_GREATER_THAN /* greater than */ } _pdc_query_op_t; -typedef enum { ROW_major, COL_major } _pdc_major_type_t; - -typedef enum { C_lang = 0, FORTRAN_lang, PYTHON_lang, JULIA_lang, N_LANGUAGES } _pdc_analysis_language_t; - /***************************/ /* Library Private Structs */ /***************************/ From 2e8323a61cd892c4a4d5fcaff58af74e3e3468b0 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Tue, 8 Aug 2023 15:51:41 -0700 Subject: [PATCH 215/216] Fix cmake path (#121) * Fix cmake path * Fix cmake path --- CMake/pdc-config.cmake.build.in | 2 +- CMake/pdc-config.cmake.install.in | 2 +- CMakeLists.txt | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CMake/pdc-config.cmake.build.in b/CMake/pdc-config.cmake.build.in index e61dd4bac..920bad2a1 100644 --- a/CMake/pdc-config.cmake.build.in +++ b/CMake/pdc-config.cmake.build.in @@ -24,5 +24,5 @@ if(NOT TARGET "pdc" AND NOT PDC_INSTALL_SKIP_TARGETS) if(NOT TARGET "mercury") include(@MERCURY_DIR@/mercury-config.cmake) endif() - include(${SELF_DIR}/api/pdc-targets.cmake) + include(@PDC_INSTALL_SHARE_DIR@/pdc-targets.cmake) endif() diff --git a/CMake/pdc-config.cmake.install.in b/CMake/pdc-config.cmake.install.in index 2e8bba491..e3ead7e94 100644 --- a/CMake/pdc-config.cmake.install.in +++ b/CMake/pdc-config.cmake.install.in @@ -24,5 +24,5 @@ if(NOT TARGET "pdc" AND NOT PDC_INSTALL_SKIP_TARGETS) if(NOT TARGET "mercury") include(@MERCURY_DIR@/mercury-config.cmake) endif() - include(${SELF_DIR}/pdc-targets.cmake) + include(@PDC_INSTALL_SHARE_DIR@/pdc-targets.cmake) endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index 715766747..315305b9b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,6 +81,9 @@ endif() if(NOT PDC_INSTALL_DATA_DIR) set(PDC_INSTALL_DATA_DIR ${CMAKE_INSTALL_PREFIX}/share) endif() +if(NOT PDC_INSTALL_SHARE_DIR) + set(PDC_INSTALL_SHARE_DIR ${CMAKE_INSTALL_PREFIX}/share/cmake/pdc) +endif() # Setting this ensures that "make install" will leave rpaths to external # libraries intact on "make install". This ensures that one can install a From c5c7b91556f54445da4e03bb636236bcf3533f60 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Tue, 15 Aug 2023 10:59:16 -0700 Subject: [PATCH 216/216] Kvtag query (#122) * Add a new collective kvtag query api to return full/aggregated results to all clients * Committing clang-format changes * Add test code * Committing clang-format changes * Add an optimization when multiple clients issue different queries * Add test program * Fix free issue * Committing clang-format changes --------- Co-authored-by: github-actions --- src/api/include/pdc_client_connect.h | 23 +- src/api/pdc_client_connect.c | 139 ++++++++++-- .../pdc_server_region_cache.c | 7 + src/tests/CMakeLists.txt | 1 + src/tests/kvtag_query_mpi.c | 207 ++++++++++++++++++ 5 files changed, 355 insertions(+), 22 deletions(-) create mode 100644 src/tests/kvtag_query_mpi.c diff --git a/src/api/include/pdc_client_connect.h b/src/api/include/pdc_client_connect.h index 933f2a2e8..3eb109714 100644 --- a/src/api/include/pdc_client_connect.h +++ b/src/api/include/pdc_client_connect.h @@ -595,16 +595,31 @@ perr_t PDC_Client_create_cont_id_mpi(const char *cont_name, pdcid_t cont_create_ perr_t PDC_Client_query_kvtag(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids); /** - * Client sends query requests to server (used by MPI mode) + * Client sends query requests to server (used by MPI mode), each client gets a subset of the + * queried results * - * \param kvtag [IN] ********* - * \param n_res [IN] ********** - * \param pdc_ids [OUT] ********* + * \param kvtag [IN] kvtag + * \param n_res [OUT] number of hits + * \param pdc_ids [OUT] object ids of hits, unordered * * \return Non-negative on success/Negative on failure */ perr_t PDC_Client_query_kvtag_col(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids); +#ifdef ENABLE_MPI +/** + * Client sends query requests to server (used by MPI mode), all clients get the same aggregated + * query results, currently assumes MPI_COMM_WORLD + * + * \param kvtag [IN] kvtag + * \param n_res [OUT] number of hits + * \param pdc_ids [OUT] object ids of hits, unordered + * + * \return Non-negative on success/Negative on failure + */ +perr_t PDC_Client_query_kvtag_mpi(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids, MPI_Comm comm); +#endif + /** * Client sends query requests to server (used by MPI mode) * diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index 34cef9ad8..34fc8415e 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -7264,8 +7264,12 @@ PDC_Client_query_kvtag_server(uint32_t server_id, const pdc_kvtag_t *kvtag, int FUNC_ENTER(NULL); - if (kvtag == NULL || n_res == NULL || out == NULL) - PGOTO_ERROR(FAIL, "==CLIENT[%d]: input is NULL!", pdc_client_mpi_rank_g); + if (kvtag == NULL) + PGOTO_ERROR(FAIL, "==CLIENT[%d]: %s - kvtag is NULL!", pdc_client_mpi_rank_g, __func__); + if (n_res == NULL) + PGOTO_ERROR(FAIL, "==CLIENT[%d]: %s - n_res is NULL!", pdc_client_mpi_rank_g, __func__); + if (out == NULL) + PGOTO_ERROR(FAIL, "==CLIENT[%d]: %s - out is NULL!", pdc_client_mpi_rank_g, __func__); if (kvtag->name == NULL) in.name = " "; @@ -7307,7 +7311,8 @@ PDC_Client_query_kvtag_server(uint32_t server_id, const pdc_kvtag_t *kvtag, int PDC_Client_check_bulk(send_context_g); *n_res = bulk_arg->n_meta; - *out = bulk_arg->obj_ids; + if (*n_res > 0) + *out = bulk_arg->obj_ids; free(bulk_arg); // TODO: need to be careful when freeing the lookup_args, as it include the results returned to user @@ -7320,18 +7325,21 @@ PDC_Client_query_kvtag_server(uint32_t server_id, const pdc_kvtag_t *kvtag, int perr_t PDC_Client_query_kvtag(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids) { - perr_t ret_value = SUCCEED; - int32_t i; - int nmeta = 0; + perr_t ret_value = SUCCEED; + int i, nmeta = 0; + uint32_t server_id; FUNC_ENTER(NULL); *n_res = 0; for (i = 0; i < pdc_server_num_g; i++) { - ret_value = PDC_Client_query_kvtag_server((uint32_t)i, kvtag, &nmeta, pdc_ids); + // when there are multiple clients issuing different queries concurrently, try to balance the + // server workload by having different clients sending queries with a different order + server_id = (pdc_client_mpi_rank_g + i) % pdc_server_num_g; + ret_value = PDC_Client_query_kvtag_server(server_id, kvtag, &nmeta, pdc_ids); if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error with PDC_Client_query_kvtag_server to server %d", - pdc_client_mpi_rank_g, i); + pdc_client_mpi_rank_g, server_id); } *n_res = nmeta; @@ -7365,14 +7373,15 @@ PDC_assign_server(uint32_t *my_server_start, uint32_t *my_server_end, uint32_t * FUNC_LEAVE_VOID; } -// All clients collectively query all servers +// All clients collectively query all servers, each client gets partial results perr_t PDC_Client_query_kvtag_col(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids) { - perr_t ret_value = SUCCEED; - int32_t my_server_start, my_server_end, my_server_count; - int32_t i; - int nmeta = 0; + perr_t ret_value = SUCCEED; + int32_t my_server_start, my_server_end, my_server_count; + int32_t i; + int nmeta = 0; + uint64_t *temp_ids = NULL; FUNC_ENTER(NULL); @@ -7392,23 +7401,117 @@ PDC_Client_query_kvtag_col(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ } } - *n_res = 0; + *n_res = 0; + *pdc_ids = NULL; for (i = my_server_start; i < my_server_end; i++) { - if (i >= pdc_server_num_g) { + if (i >= pdc_server_num_g) break; - } - ret_value = PDC_Client_query_kvtag_server((uint32_t)i, kvtag, &nmeta, pdc_ids); + + ret_value = PDC_Client_query_kvtag_server((uint32_t)i, kvtag, &nmeta, &temp_ids); if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error with PDC_Client_query_kvtag_server to server %u", pdc_client_mpi_rank_g, i); + if (i == my_server_start) + *pdc_ids = temp_ids; + else { + *pdc_ids = (uint64_t *)realloc(*pdc_ids, sizeof(uint64_t) * (*n_res + nmeta)); + memcpy(*pdc_ids + (*n_res) * sizeof(uint64_t), temp_ids, nmeta * sizeof(uint64_t)); + if (temp_ids) + free(temp_ids); + } + *n_res = *n_res + nmeta; } - *n_res = nmeta; +done: + fflush(stdout); + FUNC_LEAVE(ret_value); +} + +#ifdef ENABLE_MPI +// All clients collectively query all servers, all clients get all results +perr_t +PDC_Client_query_kvtag_mpi(const pdc_kvtag_t *kvtag, int *n_res, uint64_t **pdc_ids, MPI_Comm comm) +{ + perr_t ret_value = SUCCEED; + int32_t my_server_start, my_server_end, my_server_count; + int32_t i; + int nmeta = 0, *all_nmeta = NULL, ntotal = 0, *disp = NULL; + uint64_t *temp_ids = NULL; + + FUNC_ENTER(NULL); + + if (pdc_server_num_g > pdc_client_mpi_size_g) { + my_server_count = pdc_server_num_g / pdc_client_mpi_size_g; + my_server_start = pdc_client_mpi_rank_g * my_server_count; + my_server_end = my_server_start + my_server_count; + if (pdc_client_mpi_rank_g == pdc_client_mpi_size_g - 1) { + my_server_end += pdc_server_num_g % pdc_client_mpi_size_g; + } + } + else { + my_server_start = pdc_client_mpi_rank_g; + my_server_end = my_server_start + 1; + if (pdc_client_mpi_rank_g >= pdc_server_num_g) { + my_server_end = 0; + } + } + + *n_res = 0; + *pdc_ids = NULL; + for (i = my_server_start; i < my_server_end; i++) { + if (i >= pdc_server_num_g) + break; + + /* printf("==PDC_CLIENT[%d]: querying server %u\n", pdc_client_mpi_rank_g, i); */ + + ret_value = PDC_Client_query_kvtag_server((uint32_t)i, kvtag, &nmeta, &temp_ids); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "==PDC_CLIENT[%d]: error in %s querying server %u", pdc_client_mpi_rank_g, + __func__, i); + if (i == my_server_start) + *pdc_ids = temp_ids; + else if (nmeta > 0) { + *pdc_ids = (uint64_t *)realloc(*pdc_ids, sizeof(uint64_t) * (*n_res + nmeta)); + memcpy(*pdc_ids + (*n_res) * sizeof(uint64_t), temp_ids, nmeta * sizeof(uint64_t)); + free(temp_ids); + } + *n_res = *n_res + nmeta; + /* printf("==PDC_CLIENT[%d]: server %u returned %d res \n", pdc_client_mpi_rank_g, i, *n_res); */ + } + + if (pdc_client_mpi_size_g == 1) + goto done; + + all_nmeta = (int *)malloc(pdc_client_mpi_size_g * sizeof(int)); + disp = (int *)malloc(pdc_client_mpi_size_g * sizeof(int)); + MPI_Allgather(n_res, 1, MPI_INT, all_nmeta, 1, MPI_INT, comm); + for (i = 0; i < pdc_client_mpi_size_g; i++) { + ntotal += all_nmeta[i]; + if (i == 0) + disp[i] = 0; + else + disp[i] = disp[i - 1] + all_nmeta[i]; + } + + /* printf("==PDC_CLIENT[%d]: after allgather \n", pdc_client_mpi_rank_g); */ + + temp_ids = (uint64_t *)malloc(ntotal * sizeof(uint64_t)); + MPI_Allgatherv(pdc_ids, *n_res, MPI_UINT64_T, temp_ids, all_nmeta, disp, MPI_UINT64_T, comm); + + /* printf("==PDC_CLIENT[%d]: after allgatherv\n", pdc_client_mpi_rank_g); */ + + free(all_nmeta); + free(disp); + if (*n_res > 0) + free(*pdc_ids); + *pdc_ids = temp_ids; + *n_res = ntotal; done: fflush(stdout); FUNC_LEAVE(ret_value); } +#endif // Delete a tag specified by a name, and whether it is from a container or an object static perr_t diff --git a/src/server/pdc_server_region/pdc_server_region_cache.c b/src/server/pdc_server_region/pdc_server_region_cache.c index d2f2abe56..0054f9485 100644 --- a/src/server/pdc_server_region/pdc_server_region_cache.c +++ b/src/server/pdc_server_region/pdc_server_region_cache.c @@ -679,6 +679,7 @@ PDC_region_cache_flush_by_pointer(uint64_t obj_id, pdc_obj_cache *obj_cache) char ** buf, **new_buf, *buf_ptr = NULL; uint64_t * start, *end, *new_start, *new_end; int merged_request_size = 0; + int server_rank = 0; uint64_t unit; struct pdc_region_info **obj_regions; #ifdef PDC_TIMING @@ -751,6 +752,9 @@ PDC_region_cache_flush_by_pointer(uint64_t obj_id, pdc_obj_cache *obj_cache) nflush += merged_request_size; } +#ifdef ENABLE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &server_rank); +#endif // Iterate through all cache regions and use POSIX I/O to write them back to file system. region_cache_iter = obj_cache->region_cache; while (region_cache_iter != NULL) { @@ -764,6 +768,9 @@ PDC_region_cache_flush_by_pointer(uint64_t obj_id, pdc_obj_cache *obj_cache) if (obj_cache->ndim >= 3) write_size *= region_cache_info->size[2]; + printf("==PDC_SERVER[%d]: server flushed %.1f / %.1f MB to storage\n", server_rank, + write_size / 1048576.0, total_cache_size / 1048576.0); + total_cache_size -= write_size; free(region_cache_info->offset); if (obj_cache->ndim > 1) { diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 2fd2d0839..ca1856b3f 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -67,6 +67,7 @@ set(PROGRAMS kvtag_add_get_scale # kvtag_query kvtag_query_scale + kvtag_query_mpi # obj_transformation region_transfer_query region_transfer diff --git a/src/tests/kvtag_query_mpi.c b/src/tests/kvtag_query_mpi.c new file mode 100644 index 000000000..22c619c97 --- /dev/null +++ b/src/tests/kvtag_query_mpi.c @@ -0,0 +1,207 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include +#include +#include +#include "pdc.h" +#include "pdc_client_connect.h" + +int +assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) +{ + if (rank > size || my_count == NULL || my_start == NULL) { + printf("assign_work_to_rank(): Error with input!\n"); + return -1; + } + if (nwork < size) { + if (rank < nwork) + *my_count = 1; + else + *my_count = 0; + (*my_start) = rank * (*my_count); + } + else { + (*my_count) = nwork / size; + (*my_start) = rank * (*my_count); + + // Last few ranks may have extra work + if (rank >= size - nwork % size) { + (*my_count)++; + (*my_start) += (rank - (size - nwork % size)); + } + } + + return 1; +} + +void +print_usage(char *name) +{ + printf("%s n_obj n_query\n", name); +} + +int +main(int argc, char *argv[]) +{ + pdcid_t pdc, cont_prop, cont, obj_prop; + pdcid_t * obj_ids; + int n_obj, n_add_tag, my_obj, my_obj_s, my_add_tag, my_add_tag_s; + int proc_num, my_rank, i, v, iter, round; + char obj_name[128]; + double stime, total_time; + pdc_kvtag_t kvtag; + uint64_t * pdc_ids; + int nres, ntotal; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &proc_num); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#endif + + if (argc < 3) { + if (my_rank == 0) + print_usage(argv[0]); + goto done; + } + n_obj = atoi(argv[1]); + round = atoi(argv[2]); + n_add_tag = n_obj / 100; + + // create a pdc + pdc = PDCinit("pdc"); + + // create a container property + cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + printf("Fail to create container property @ line %d!\n", __LINE__); + + // create a container + cont = PDCcont_create("c1", cont_prop); + if (cont <= 0) + printf("Fail to create container @ line %d!\n", __LINE__); + + // create an object property + obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); + if (obj_prop <= 0) + printf("Fail to create object property @ line %d!\n", __LINE__); + + // Create a number of objects, add at least one tag to that object + assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); + if (my_rank == 0) + printf("I will create %d obj\n", my_obj); + obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + for (i = 0; i < my_obj; i++) { + sprintf(obj_name, "obj%d", my_obj_s + i); + obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); + if (obj_ids[i] <= 0) { + printf("Fail to create object @ line %d!\n", __LINE__); + goto done; + } + } + + if (my_rank == 0) + printf("Created %d objects\n", n_obj); + fflush(stdout); + + // Add tags + kvtag.name = "Group"; + kvtag.value = (void *)&v; + kvtag.type = PDC_INT; + kvtag.size = sizeof(int); + + for (iter = 0; iter < round; iter++) { + assign_work_to_rank(my_rank, proc_num, n_add_tag, &my_add_tag, &my_add_tag_s); + + v = iter; + for (i = 0; i < my_add_tag; i++) { + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) { + printf("fail to add a kvtag to o%d\n", i + my_obj_s); + goto done; + } + } + + if (my_rank == 0) + printf("Rank %d: Added a kvtag to %d objects\n", my_rank, my_add_tag); + fflush(stdout); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + n_add_tag *= 2; + } + + n_add_tag = n_obj / 100; + + for (iter = 0; iter < round; iter++) { + v = iter; + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + + if (PDC_Client_query_kvtag_mpi(&kvtag, &nres, &pdc_ids, MPI_COMM_WORLD) < 0) { + printf("fail to query kvtag [%s] with rank %d\n", kvtag.name, my_rank); + break; + } + + if (nres != n_add_tag) + printf("Rank %d: query result %d doesn't match expected %d\n", my_rank, nres, n_add_tag); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) + printf("Total time to query %d objects with tag: %.5e\n", nres, total_time); + fflush(stdout); + n_add_tag *= 2; + } + + // close a container + if (PDCcont_close(cont) < 0) + printf("fail to close container c1\n"); + + // close an object property + if (PDCprop_close(obj_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + // close a container property + if (PDCprop_close(cont_prop) < 0) + printf("Fail to close property @ line %d\n", __LINE__); + + // close pdc + if (PDCclose(pdc) < 0) + printf("fail to close PDC\n"); +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return 0; +}