Add a patch to ROMIO to print the number of calls to memcpy()

This is to show the excessive number of calls to memcpy() in macro ADIOI_BUF_COPY of subroutine ADIOI_Fill_send_buffer() in file mpich/src/mpi/romio/adio/common/ad_write_coll.c Run pio_noncontig with the following commands and compare. % mpiexec -n 16 pio_noncontig -k 256 -c 32768 -w Number of global variables = 64 Each global variable is of size 256 x 32768 bytes Each local variable is of size 256 x 16 bytes Gap between the first 2 variables is of size 16 bytes Number of subarray types concatenated is 8192 Each process makes a request of amount 33554688 bytes ROMIO hint set: cb_buffer_size = 1048576 ROMIO hint set: cb_nodes = 4 rank 0: ADIOI_GEN_WriteStridedColl number of memcpy called = 2097153 rank 1: ADIOI_GEN_WriteStridedColl number of memcpy called = 2097153 rank 2: ADIOI_GEN_WriteStridedColl number of memcpy called = 2097153 rank 3: ADIOI_GEN_WriteStridedColl number of memcpy called = 2097153 --------------------------------------------------------- Time of collective write = 30.83 sec --------------------------------------------------------- % mpiexec -n 16 pio_noncontig -k 256 -c 32768 -w -g 0 Number of global variables = 64 Each global variable is of size 256 x 32768 bytes Each local variable is of size 256 x 16 bytes Gap between the first 2 variables is of size 0 bytes Number of subarray types concatenated is 8192 Each process makes a request of amount 33554688 bytes ROMIO hint set: cb_buffer_size = 1048576 ROMIO hint set: cb_nodes = 4 rank 0: ADIOI_GEN_WriteStridedColl number of memcpy called = 0 rank 1: ADIOI_GEN_WriteStridedColl number of memcpy called = 0 rank 2: ADIOI_GEN_WriteStridedColl number of memcpy called = 0 rank 3: ADIOI_GEN_WriteStridedColl number of memcpy called = 0 --------------------------------------------------------- Time of collective write = 9.27 sec ---------------------------------------------------------
wkliao · Apr 18, 2024 · e3a464e · e3a464e
1 parent 7426408
commit e3a464e
Showing 1 changed file with 50 additions and 0 deletions.
diff --git a/tests/0001-print-number-of-calls-to-memcpy.patch b/tests/0001-print-number-of-calls-to-memcpy.patch
@@ -0,0 +1,50 @@
+From 82487c29f5737b8f4c959a8dfc148d52deed0c51 Mon Sep 17 00:00:00 2001
+From: wkliao <wkliao@users.noreply.github.com>
+Date: Thu, 18 Apr 2024 15:44:46 -0500
+Subject: [PATCH] print number of calls to memcpy()
+
+---
+ src/mpi/romio/adio/common/ad_write_coll.c | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/src/mpi/romio/adio/common/ad_write_coll.c b/src/mpi/romio/adio/common/ad_write_coll.c
+index 2b7ed69f..84c5875d 100644
+--- a/src/mpi/romio/adio/common/ad_write_coll.c
++++ b/src/mpi/romio/adio/common/ad_write_coll.c
+@@ -10,6 +10,8 @@
+ #include "mpe.h"
+ #endif
+
++static int num_memcpy;
++
+ /* prototypes of functions used for collective writes only. */
+ static void ADIOI_Exch_and_write(ADIO_File fd, void *buf, MPI_Datatype
+ datatype, int nprocs, int myrank,
+@@ -78,6 +80,7 @@ void ADIOI_GEN_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
+ ADIO_Offset *len_list = NULL;
+ int old_error, tmp_error;
+
++num_memcpy = 0;
+ if (fd->hints->cb_pfr != ADIOI_HINT_DISABLE) {
+ /* Cast away const'ness as the below function is used for read
+ * and write */
+@@ -247,6 +250,8 @@ void ADIOI_GEN_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
+ #ifdef AGGREGATION_PROFILE
+ MPE_Log_event(5013, 0, NULL);
+ #endif
++
++if (fd->is_agg) printf("rank %d: %s number of memcpy called = %d\n",myrank,__func__,num_memcpy);
+ }
+
+
+@@ -791,6 +796,7 @@ static void ADIOI_W_Exchange_data(ADIO_File fd, void *buf, char *write_buf,
+ ADIOI_Assert(size_in_buf == (size_t)size_in_buf); \
+ memcpy(&(send_buf[p][send_buf_idx[p]]), \
+ ((char *) buf) + user_buf_idx, size_in_buf); \
++num_memcpy++; \
+ send_buf_idx[p] += size_in_buf; \
+ user_buf_idx += size_in_buf; \
+ flat_buf_sz -= size_in_buf; \
+-- 
+2.39.3
+