diff --git a/configure.ac b/configure.ac index 6964e7f9639..324704ec95b 100644 --- a/configure.ac +++ b/configure.ac @@ -52,6 +52,22 @@ if test "x$enable_logging" = xyes; then AC_DEFINE([PIO_ENABLE_LOGGING], 1, [If true, turn on logging.]) fi +# Does the user want to use MPE library? +AC_MSG_CHECKING([whether use of MPE library is enabled]) +AC_ARG_ENABLE([mpe], + [AS_HELP_STRING([--enable-mpe], + [enable use of MPE library for timing and diagnostic info (may negatively impact performance).])]) +test "x$enable_mpe" = xyes || enable_mpe=no +AC_MSG_RESULT([$enable_mpe]) +if test "x$enable_mpe" = xyes; then + AC_SEARCH_LIBS([MPE_Log_get_event_number], [mpe], [HAVE_LIBMPE=yes], [HAVE_LIBMPE=no], [-lpthread -lm]) + AC_CHECK_HEADERS([mpe.h], [HAVE_MPE=yes], [HAVE_MPE=no]) + if test "x$HAVE_LIBMPE" = xno -o "x$HAVE_MPE" = xno; then + AC_MSG_ERROR([MPE not found but --enable-mpe used.]) + fi + AC_DEFINE([USE_MPE], 1, [If true, use MPE timing library.]) +fi + # Does the user want to enable timing? AC_MSG_CHECKING([whether GPTL timing library is used]) AC_ARG_ENABLE([timing], diff --git a/tests/cunit/test_perf2.c b/tests/cunit/test_perf2.c index 858623bce9b..e5c7fbad7e7 100644 --- a/tests/cunit/test_perf2.c +++ b/tests/cunit/test_perf2.c @@ -9,6 +9,9 @@ #include #include #include +#ifdef USE_MPE +#include +#endif /* USE_MPE */ /* The number of tasks this test should run on. */ #define TARGET_NTASKS 16 @@ -30,9 +33,12 @@ #define NDIM3 3 /* The length of our sample data along each dimension. */ -#define X_DIM_LEN 1024 -#define Y_DIM_LEN 1024 -#define Z_DIM_LEN 128 +#define X_DIM_LEN 128 +#define Y_DIM_LEN 128 +#define Z_DIM_LEN 32 +/* #define X_DIM_LEN 1024 */ +/* #define Y_DIM_LEN 1024 */ +/* #define Z_DIM_LEN 128 */ /* The number of timesteps of data to write. */ #define NUM_TIMESTEPS 10 @@ -59,6 +65,64 @@ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN}; /* Run test for each of the rearrangers. */ #define NUM_REARRANGERS_TO_TEST 2 +#ifdef USE_MPE +/* These are for the event numbers array used to log various events in + * the program with the MPE library, which produces output for the + * Jumpshot program. */ +#define NUM_EVENTS 7 +#define START 0 +#define END 1 +#define INIT 0 +#define DECOMP 1 +#define CREATE 2 +#define DARRAY_WRITE 3 +#define CLOSE 4 +#define CALCULATE 5 +#define INGEST 6 + +#define ERR_LOGGING 99 + +/* This array holds even numbers for MPE. */ +int event_num[2][NUM_EVENTS]; + +/* This will set up the MPE logging event numbers. */ +int +init_logging(int my_rank, int event_num[][NUM_EVENTS]) +{ + /* Get a bunch of event numbers. */ + event_num[START][INIT] = MPE_Log_get_event_number(); + event_num[END][INIT] = MPE_Log_get_event_number(); + event_num[START][DECOMP] = MPE_Log_get_event_number(); + event_num[END][DECOMP] = MPE_Log_get_event_number(); + event_num[START][INGEST] = MPE_Log_get_event_number(); + event_num[END][INGEST] = MPE_Log_get_event_number(); + event_num[START][CLOSE] = MPE_Log_get_event_number(); + event_num[END][CLOSE] = MPE_Log_get_event_number(); + event_num[START][CALCULATE] = MPE_Log_get_event_number(); + event_num[END][CALCULATE] = MPE_Log_get_event_number(); + event_num[START][CREATE] = MPE_Log_get_event_number(); + event_num[END][CREATE] = MPE_Log_get_event_number(); + event_num[START][DARRAY_WRITE] = MPE_Log_get_event_number(); + event_num[END][DARRAY_WRITE] = MPE_Log_get_event_number(); + + /* You should track at least initialization and partitioning, data + * ingest, update computation, all communications, any memory + * copies (if you do that), any output rendering, and any global + * communications. */ + if (!my_rank) + { + MPE_Describe_state(event_num[START][INIT], event_num[END][INIT], "init", "yellow"); + MPE_Describe_state(event_num[START][INGEST], event_num[END][INGEST], "ingest", "red"); + MPE_Describe_state(event_num[START][DECOMP], event_num[END][DECOMP], "decomposition", "green"); + MPE_Describe_state(event_num[START][CALCULATE], event_num[END][CALCULATE], "calculate", "orange"); + MPE_Describe_state(event_num[START][CREATE], event_num[END][CREATE], "create", "purple"); + MPE_Describe_state(event_num[START][CLOSE], event_num[END][CLOSE], "close file", "blue"); + MPE_Describe_state(event_num[START][DARRAY_WRITE], event_num[END][DARRAY_WRITE], "darray write", "pink"); + } + return 0; +} +#endif /* USE_MPE */ + /* Create the decomposition to divide the 4-dimensional sample data * between the 4 tasks. For the purposes of decomposition we are only * concerned with 3 dimensions - we ignore the unlimited dimension. @@ -147,6 +211,11 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, float delta_in_sec; float mb_per_sec; +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[START][CREATE], 0, "start init"))) + return ERR_MPI; +#endif /* USE_MPE */ + /* Create the filename. Use the same filename for all, so we * don't waste disk space. */ /* sprintf(filename, "data_%s_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[fmt], */ @@ -174,6 +243,11 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, if ((ret = PIOc_enddef(ncid))) ERR(ret); +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[END][CREATE], 0, "end init"))) + MPIERR(ret); +#endif /* USE_MPE */ + /* Start the clock. */ gettimeofday(&starttime, NULL); @@ -183,6 +257,11 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, for (int f = 0; f < arraylen; f++) test_data[f] = (my_rank * 10 + f) + t * 1000; +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[START][DARRAY_WRITE], 0, "start init"))) + return ERR_MPI; +#endif /* USE_MPE */ + /* Set the value of the record dimension. */ if ((ret = PIOc_setframe(ncid, varid, t))) ERR(ret); @@ -191,13 +270,29 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) ERR(ret); +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[END][DARRAY_WRITE], 0, "end init"))) + MPIERR(ret); +#endif /* USE_MPE */ + num_megabytes += (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * sizeof(int))/(1024*1024); } +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[START][CLOSE], 0, "start init"))) + return ERR_MPI; +#endif /* USE_MPE */ + /* Close the netCDF file. */ if ((ret = PIOc_closefile(ncid))) ERR(ret); +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[END][CLOSE], 0, "end init"))) + MPIERR(ret); +#endif /* USE_MPE */ + + /* Stop the clock. */ gettimeofday(&endtime, NULL); @@ -330,6 +425,11 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, if ((ret = MPI_Comm_size(test_comm, &my_test_size))) MPIERR(ret); +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[START][DECOMP], 0, "start init"))) + return ERR_MPI; +#endif /* USE_MPE */ + /* Decompose the data over the tasks. */ if ((ret = create_decomposition_3d(ntasks, my_rank, iosysid, &ioid))) return ret; @@ -339,6 +439,11 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, ntasks, rearranger, test_comm))) return ret; +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[END][DECOMP], 0, "end init"))) + MPIERR(ret); +#endif /* USE_MPE */ + /* Test with/without providing a fill value to PIOc_write_darray(). */ for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) { @@ -348,10 +453,20 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, return ret; } +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[START][DECOMP], 0, "start init"))) + return ERR_MPI; +#endif /* USE_MPE */ + /* Free the PIO decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[END][DECOMP], 0, "end init"))) + MPIERR(ret); +#endif /* USE_MPE */ + return PIO_NOERR; } @@ -377,6 +492,15 @@ int main(int argc, char **argv) 0, -1, &test_comm))) ERR(ERR_INIT); +#ifdef USE_MPE + + if ((ret = MPE_Init_log())) + return ret; + if (init_logging(my_rank, event_num)) + return ERR_LOGGING; +#endif /* USE_MPE */ + + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) return ret; @@ -404,12 +528,22 @@ int main(int argc, char **argv) /* for (r = 0; r < NUM_REARRANGERS_TO_TEST; r++) */ for (r = 0; r < 1; r++) { +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[START][INIT], 0, "start init"))) + return ERR_MPI; +#endif /* USE_MPE */ + /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ if ((ret = PIOc_Init_Intracomm(test_comm, num_io_procs[i], ioproc_stride, ioproc_start, rearranger[r], &iosysid))) return ret; +#ifdef USE_MPE + if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init"))) + MPIERR(ret); +#endif /* USE_MPE */ + /* Run tests. */ if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, ntasks, num_io_procs[i], rearranger[r], test_comm))) @@ -418,10 +552,23 @@ int main(int argc, char **argv) /* Finalize PIO system. */ if ((ret = PIOc_free_iosystem(iosysid))) return ret; - } /* next rearranger */ } /* next num io procs */ + +#ifdef USE_MPE + { + /* This causes problems on my MPICH2 library on Linux, but seems to be + * required for frost. */ + char file_name[128]; + sprintf(file_name, "chart_%d", 1); + if ((ret = MPE_Finish_log(file_name))) + MPIERR(ret); + } + +#endif /* USE_MPE */ + + if (!my_rank) printf("finalizing io_test!\n");