#2302: initialize and call papi in lb data

src/vt/context/context.cc

@@ -90,6 +90,23 @@ Context::Context([[maybe_unused]] bool const is_interop, MPI_Comm comm) {
   communicator_ = comm;
   numNodes_ = static_cast<NodeType>(numNodesLocal);
   thisNode_ = static_cast<NodeType>(thisNodeLocal);
+
+  int retval;
+
+  /* Initialize the PAPI library */
+  retval = PAPI_library_init(PAPI_VER_CURRENT);
+  if (retval != PAPI_VER_CURRENT)
+      handle_papi_error(retval);
+
+  /* Check for possible failures */
+  if (retval < 0)
+      handle_papi_error(retval);
+
+  /* Print PAPI Version */
+  fprintf(stdout, "PAPI Version Number\n");
+  fprintf(stdout, "MAJOR:    %d\n", PAPI_VERSION_MAJOR(retval));
+  fprintf(stdout, "MINOR:    %d\n", PAPI_VERSION_MINOR(retval));
+  fprintf(stdout, "REVISION: %d\n", PAPI_VERSION_REVISION(retval));
 }
 
 Context::~Context() {

src/vt/context/context.h

@@ -46,6 +46,7 @@
 
 #include <memory>
 #include <mpi.h>
+#include <papi.h>
 
 #include "vt/config.h"
 #include "vt/runtime/component/component_pack.h"
@@ -124,6 +125,12 @@ struct Context : runtime::component::Component<Context> {
 
   std::string name() override { return "Context"; }
 
+  void handle_papi_error (int retval)
+  {
+    printf("PAPI error %d: %s\n", retval, PAPI_strerror(retval));
+    exit(1);
+  }
+
   template <typename SerializerT>
   void serialize(SerializerT& s) {
     s | thisNode_

src/vt/context/runnable_context/lb_data.cc

@@ -51,13 +51,53 @@ void LBData::start(TimeType time) {
   if (should_instrument_) {
     lb_data_->start(time);
   }
+
+  /* -- PAPI START -- */
+
+  /* Start counting events in the Event Set */
+  // papi_retval_ = PAPI_start(EventSet_);
+  // if (papi_retval_ != PAPI_OK)
+  //   handle_papi_error(papi_retval_, "LBData start: Starting counting events in the Event Set: ");
+
+  /* Gets the starting time in clock cycles */
+  start_cycles_ = PAPI_get_real_cyc();
+
+  /* Gets the starting time in microseconds */
+  start_usec_ = PAPI_get_real_usec();
+
+  /* ---------------- */
 }
 
 void LBData::finish(TimeType time) {
   // record end time
   if (should_instrument_) {
     lb_data_->stop(time);
   }
+
+  /* -- PAPI READ AND STOP -- */
+
+  /* Read the counting events in the Event Set */
+  // papi_retval_ = PAPI_read(EventSet_, papi_values_);
+  // if (papi_retval_ != PAPI_OK)
+  //   handle_papi_error(papi_retval_, "LBData finish: Reading the counting events in the Event Set: ");
+
+  // printf("Counters after LBData::finish: %lld\n",papi_values_[0]);
+
+  // /* Stop the counting of events in the Event Set */
+  // papi_retval_ = PAPI_stop(EventSet_, papi_values_);
+  // if (papi_retval_ != PAPI_OK)
+  //   handle_papi_error(papi_retval_, "LBData finish: Stoping the counting of events in the Event Set: ");
+
+  /* Gets the ending time in clock cycles */
+  end_cycles_ = PAPI_get_real_cyc();
+
+  /* Gets the ending time in microseconds */
+  end_usec_ = PAPI_get_real_usec();
+
+  printf("Wall clock cycles: %lld\n", end_cycles_ - start_cycles_);
+  printf("Wall clock time in microseconds: %lld\n", end_usec_ - start_usec_);
+
+  /* ------------------------ */
 }
 
 void LBData::send(elm::ElementIDStruct dest, MsgSizeType bytes) {

src/vt/context/runnable_context/lb_data.h

@@ -47,6 +47,8 @@
 #include "vt/vrt/collection/balance/lb_common.h"
 #include "vt/elm/elm_lb_data.fwd.h"
 
+#include <papi.h>
+
 namespace vt { namespace ctx {
 
 /**
@@ -58,6 +60,12 @@ struct LBData {
   using ElementIDStruct = elm::ElementIDStruct;
   using ElementLBData    = elm::ElementLBData;
 
+  void handle_papi_error (int retval, std::string info)
+  {
+    printf("%s: PAPI error %d: %s\n", info.c_str(), retval, PAPI_strerror(retval));
+    exit(1);
+  }
+
   LBData() = default;
 
   /**
@@ -79,7 +87,17 @@ struct LBData {
     : lb_data_(in_lb_data),
       cur_elm_id_(in_elm_id),
       should_instrument_(true)
-  { }
+  {
+    /* Create the PAPI Event Set */
+    papi_retval_ = PAPI_create_eventset(&EventSet_);
+    if (papi_retval_ != PAPI_OK)
+      handle_papi_error(papi_retval_, "LBData Constructor 2: Creating the PAPI Event Set: ");
+
+    // /* Add Total Instructions Executed to the PAPI Event Set */
+    // papi_retval_ = PAPI_add_event(EventSet_, PAPI_DP_OPS);
+    // if (papi_retval_ != PAPI_OK)
+    //   handle_papi_error(papi_retval_, "LBData Constructor 2: Adding Total Instructions Executed to the PAPI Event Set: ");
+  }
 
   /**
    * \brief Return whether time is required
@@ -119,6 +137,10 @@ struct LBData {
   ElementLBData* lb_data_ = nullptr;     /**< Element LB data */
   ElementIDStruct cur_elm_id_ = {};   /**< Current element ID  */
   bool should_instrument_ = false;    /**< Whether we are instrumenting */
+  long_long papi_values_[1];
+  int EventSet_ = PAPI_NULL;
+  int papi_retval_;
+  long long start_cycles_, end_cycles_, start_usec_, end_usec_;
 };
 
 }} /* end namespace vt::ctx */

src/vt/context/runnable_context/lb_data.impl.h

@@ -50,6 +50,7 @@
 #include "vt/vrt/collection/manager.h"
 
 #include <memory>
+#include <papi.h>
 
 namespace vt { namespace ctx {
 
@@ -61,6 +62,20 @@ LBData::LBData(ElmT* in_elm, MsgT* msg)
 {
   // record the communication LB data right away!
   theCollection()->recordLBData(in_elm, msg);
+
+  // /* Create the PAPI Event Set */
+  // papi_retval_ = PAPI_create_eventset(&EventSet_);
+  // if (papi_retval_ != PAPI_OK) {
+  //   printf("LBData Constructor 1: Creating the PAPI Event Set: PAPI error %d: %s\n", papi_retval_, PAPI_strerror(papi_retval_));
+  //   exit(1);
+  // }
+
+  // /* Add Total Instructions Executed to the PAPI Event Set */
+  // papi_retval_ = PAPI_add_event(EventSet_, PAPI_TOT_INS);
+  // if (papi_retval_ != PAPI_OK) {
+  //   printf("LBData Constructor 1: Adding Total Instructions Executed to the PAPI Event Set: PAPI error %d: %s\n", papi_retval_, PAPI_strerror(papi_retval_));
+  //   exit(1);
+  // }
 }
 
 }} /* end namespace vt::ctx */