Sync develop to stable (v.0.5) (#201)

* Update getting_started.rst (#184) * Removing gres option for ctest (#182) * Removing gres option for ctest * Removing gres option from scripts * Update check for core --------- Co-authored-by: Hyunju Oh <hjoh16@login15.chn.perlmutter.nersc.gov> Co-authored-by: Jean Luca Bez <jlbez@lbl.gov> * enable cache by default (#187) * Removing PDC macro (#189) * Removing gres option for ctest * Removing gres option from scripts * Update check for core * Remove PDC macro * Committing clang-format changes --------- Co-authored-by: Hyunju Oh <hjoh16@login15.chn.perlmutter.nersc.gov> Co-authored-by: Jean Luca Bez <jlbez@lbl.gov> Co-authored-by: github-actions <github-actions[bot]@users.noreply.github.com> * BDCATS fix (#193) * Fix issues with bdcats_batch * Committing clang-format changes --------- Co-authored-by: github-actions <github-actions[bot]@users.noreply.github.com> * Update mpi_test.sh (#197) * Update .gitlab-ci.yml (#195) * Updates for latest integration with Jacamar and Gitlab tokens in CI * VPICIO bugfix (#196) * Fix VPICIO bug * Add more checks and error out when no server is selected * Committing clang-format changes * Add VPICIO and BDCATS to MPI test --------- Co-authored-by: github-actions <github-actions[bot]@users.noreply.github.com> Co-authored-by: Jean Luca Bez <jlbez@lbl.gov> * Fix vpicio_mts (#199) --------- Co-authored-by: Houjun Tang <htang4@lbl.gov> Co-authored-by: Hyunju Oh <oh.693@osu.edu> Co-authored-by: Hyunju Oh <hjoh16@login15.chn.perlmutter.nersc.gov> Co-authored-by: github-actions <github-actions[bot]@users.noreply.github.com>
hpc-io · Jul 1, 2024 · bdeef59 · bdeef59
1 parent a094446
commit bdeef59
Show file tree

Hide file tree

Showing 21 changed files with 392 additions and 188 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -9,7 +9,12 @@ stages:
 
 perlmutter-no-cache-build:
     stage: build
-    when: manual
+    rules:
+    - if: '$METRICS == null'
+      when: manual
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     allow_failure: false
     tags:
         - perlmutter
@@ -32,7 +37,12 @@ perlmutter-no-cache-build:
 
 perlmutter-cache-build:
     stage: build
-    when: manual
+    rules:
+    - if: '$METRICS == null'
+      when: manual
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     allow_failure: false
     tags:
         - perlmutter
@@ -60,6 +70,9 @@ perlmutter-cache-build:
 
 perlmutter-no-cache-parallel-pdc:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -80,6 +93,9 @@ perlmutter-no-cache-parallel-pdc:
 
 perlmutter-no-cache-parallel-obj:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -101,6 +117,9 @@ perlmutter-no-cache-parallel-obj:
 
 perlmutter-no-cache-parallel-cont:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -122,6 +141,9 @@ perlmutter-no-cache-parallel-cont:
 
 perlmutter-no-cache-parallel-prop:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -143,6 +165,9 @@ perlmutter-no-cache-parallel-prop:
 
 perlmutter-no-cache-parallel-region:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -164,6 +189,9 @@ perlmutter-no-cache-parallel-region:
 
 perlmutter-no-cache-parallel-region-all:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -190,6 +218,9 @@ perlmutter-no-cache-parallel-region-all:
 
 perlmutter-cache-parallel-pdc:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -210,6 +241,9 @@ perlmutter-cache-parallel-pdc:
 
 perlmutter-cache-parallel-obj:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -231,6 +265,9 @@ perlmutter-cache-parallel-obj:
 
 perlmutter-cache-parallel-cont:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -252,6 +289,9 @@ perlmutter-cache-parallel-cont:
 
 perlmutter-cache-parallel-prop:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -273,6 +313,9 @@ perlmutter-cache-parallel-prop:
 
 perlmutter-cache-parallel-region:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -294,6 +337,9 @@ perlmutter-cache-parallel-region:
 
 perlmutter-cache-parallel-region-all:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -313,26 +359,54 @@ perlmutter-cache-parallel-region-all:
         - ctest -L parallel_region_transfer_all
         - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC}
 
+perlmutter-metrics-build:
+    stage: build
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
+    rules:
+        - if: '$METRICS == "true"'
+    allow_failure: false
+    tags:
+        - perlmutter
+    variables:
+        SCHEDULER_PARAMETERS: "-A m2621 --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
+        SUPERCOMPUTER: "perlmutter"
+        MERCURY_DIR: "/global/cfs/cdirs/m2621/pdc-perlmutter/mercury/install"
+    script:
+        - module load libfabric/1.15.2.0
+        - module list
+        - mkdir -p ${PDC_BUILD_PATH}/perlmutter/metrics
+        - cd ${PDC_BUILD_PATH}/perlmutter/metrics
+        - cmake ../../.. -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DPDC_SERVER_CACHE=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=cc -DMPI_RUN_CMD="srun -A m2621 --qos=debug --constraint=cpu --tasks-per-node=64" -DCMAKE_INSTALL_PREFIX=${PDC_INSTALL_PATH}/perlmutter/metrics
+        - make -j
+        - make install
+    artifacts:
+        paths:
+            - ${PDC_BUILD_PATH}/perlmutter/metrics
+            - ${PDC_INSTALL_PATH}/perlmutter/metrics
+
 perlmutter-metrics:
     stage: metrics
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
         - if: '$METRICS == "true"'
     needs:
-        - perlmutter-cache-build
+        - perlmutter-metrics-build
     tags:
         - perlmutter
     variables:
-        PDC_N_NODES: 64
-        PDC_N_CLIENTS: 127
-        SCHEDULER_PARAMETERS: "-A m2621 --qos=regular --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00"
+        SCHEDULER_PARAMETERS: "-A m2621 --qos=${PDC_QUEUE} --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/m2621/pdc-perlmutter/mercury/install"
         PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-metrics"
         PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-metrics"
         PDC_CLIENT_LOOKUP: "NONE"
-        PDC_SERVER: "${PDC_BUILD_PATH}/perlmutter/cache/bin/pdc_server.exe"
-        PDC_SERVER_CLOSE: "${PDC_BUILD_PATH}/perlmutter/cache/bin/close_server"
-        PDC_CLIENT: "${PDC_BUILD_PATH}/perlmutter/cache/bin/vpicio_mts"
+        PDC_SERVER: "${PDC_BUILD_PATH}/perlmutter/metrics/bin/pdc_server.exe"
+        PDC_SERVER_CLOSE: "${PDC_BUILD_PATH}/perlmutter/metrics/bin/close_server"
+        PDC_CLIENT: "${PDC_BUILD_PATH}/perlmutter/metrics/bin/vpicio_mts"
         PDC_JOB_OUTPUT: "pdc-metrics.log"
     script: 
         - hostname

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -380,7 +380,7 @@ endif()
 #-----------------------------------------------------------------------------
 # SERVER CACHE option
 #-----------------------------------------------------------------------------
-option(PDC_SERVER_CACHE "Enable Server Caching." OFF)
+option(PDC_SERVER_CACHE "Enable Server Caching." ON)
 if(PDC_SERVER_CACHE)
   set(PDC_SERVER_CACHE 1)
   set(PDC_SERVER_CACHE_MAX_GB "3" CACHE STRING "Max GB for server cache")

diff --git a/examples/Makefile b/examples/Makefile
@@ -4,7 +4,7 @@ CFLAGS = -O2 -Wall -Wextra
 INCLUDES = -I. -I$(PDC_DIR)/include
 LDFLAGS = -Wl,-rpath,$(PDC_DIR)/lib -L$(PDC_DIR)/lib
 
-LIBS = -lpdc
+LIBS = -lpdc -lpdc_commons
 
 .c.o:
 	$(CC) $(CFLAGS) $(INCLUDES) -c $<

diff --git a/examples/bdcats.c b/examples/bdcats.c
@@ -101,43 +101,75 @@ main(int argc, char **argv)
     if (cont_id == 0)
         printf("Fail to create container @ line  %d!\n", __LINE__);
 
-    // open objects
-    obj_xx = PDCobj_open("obj-var-xx", pdc_id);
+        // open objects
+#ifdef ENABLE_MPI
+    obj_xx = PDCobj_open_col("obj-var-xx", pdc_id);
+#else
+    obj_xx   = PDCobj_open("obj-var-xx", pdc_id);
+#endif
     if (obj_xx == 0) {
         printf("Error when open object %s\n", "obj-var-xx");
         exit(-1);
     }
-    obj_yy = PDCobj_open("obj-var-yy", pdc_id);
+#ifdef ENABLE_MPI
+    obj_yy = PDCobj_open_col("obj-var-yy", pdc_id);
+#else
+    obj_yy   = PDCobj_open("obj-var-xx", pdc_id);
+#endif
     if (obj_yy == 0) {
         printf("Error when open object %s\n", "obj-var-yy");
         exit(-1);
     }
-    obj_zz = PDCobj_open("obj-var-zz", pdc_id);
+#ifdef ENABLE_MPI
+    obj_zz = PDCobj_open_col("obj-var-zz", pdc_id);
+#else
+    obj_zz   = PDCobj_open("obj-var-xx", pdc_id);
+#endif
     if (obj_zz == 0) {
         printf("Error when open object %s\n", "obj-var-zz");
         exit(-1);
     }
-    obj_pxx = PDCobj_open("obj-var-pxx", pdc_id);
+#ifdef ENABLE_MPI
+    obj_pxx = PDCobj_open_col("obj-var-pxx", pdc_id);
+#else
+    obj_pxx  = PDCobj_open("obj-var-xx", pdc_id);
+#endif
     if (obj_pxx == 0) {
         printf("Error when open object %s\n", "obj-var-pxx");
         exit(-1);
     }
-    obj_pyy = PDCobj_open("obj-var-pyy", pdc_id);
+#ifdef ENABLE_MPI
+    obj_pyy = PDCobj_open_col("obj-var-pyy", pdc_id);
+#else
+    obj_pyy  = PDCobj_open("obj-var-xx", pdc_id);
+#endif
     if (obj_pyy == 0) {
         printf("Error when open object %s\n", "obj-var-pyy");
         exit(-1);
     }
-    obj_pzz = PDCobj_open("obj-var-pzz", pdc_id);
+#ifdef ENABLE_MPI
+    obj_pzz = PDCobj_open_col("obj-var-pzz", pdc_id);
+#else
+    obj_pzz  = PDCobj_open("obj-var-xx", pdc_id);
+#endif
     if (obj_pzz == 0) {
         printf("Error when open object %s\n", "obj-var-pzz");
         exit(-1);
     }
-    obj_id11 = PDCobj_open("id11", pdc_id);
+#ifdef ENABLE_MPI
+    obj_id11 = PDCobj_open_col("id11", pdc_id);
+#else
+    obj_id11 = PDCobj_open("obj-var-xx", pdc_id);
+#endif
     if (obj_id11 == 0) {
         printf("Error when open object %s\n", "id11");
         exit(-1);
     }
-    obj_id22 = PDCobj_open("id22", pdc_id);
+#ifdef ENABLE_MPI
+    obj_id22 = PDCobj_open_col("id22", pdc_id);
+#else
+    obj_id22 = PDCobj_open("obj-var-xx", pdc_id);
+#endif
     if (obj_id22 == 0) {
         printf("Error when open object %s\n", "id22");
         exit(-1);