profiling: add caliper support

configure --enable-caliper to build flux with profiling

configure.ac

@@ -69,6 +69,17 @@ X_AC_CHECK_COND_LIB(dl, dlerror)
 X_AC_MALLOC
 AC_CHECK_LIB(m, floor)
 
+LIBCALIPER=
+AC_ARG_ENABLE(caliper,
+ [ --enable-caliper[=OPTS] Use caliper for profiling. [default=no] [OPTS=no/yes]], ,
+ [enable_caliper="no"])
+
+if test "$enable_caliper" = "yes"; then
+ AC_SUBST([LIBCALIPER], ["-lcaliper"])
+ AC_DEFINE([HAVE_CALIPER], [1], [Define if you have libcaliper])
+ AC_DEFINE([PROFILING], [1], [Compile in profiling hooks])
+fi
+
 AC_MSG_CHECKING(--enable-python argument)
 AC_ARG_ENABLE(python,
  [ --enable-python[=OPTS] Include Python bindings. [default=yes] [OPTS=no/yes]], ,

src/broker/broker.c

@@ -54,6 +54,10 @@
  #error gperftools headers not configured
 #endif
 #endif /* WITH_TCMALLOC */
+#if HAVE_CALIPER
+#include <caliper/cali.h>
+#include <sys/syscall.h>
+#endif
 
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/xzmalloc.h"
@@ -247,6 +251,7 @@ int main (int argc, char *argv[])
  int security_set = 0;
  int e;
 
+
  memset (&ctx, 0, sizeof (ctx));
  log_init (argv[0]);
 
@@ -431,6 +436,14 @@ int main (int argc, char *argv[])
  if (attr_set_flags (ctx.attrs, "session-id", FLUX_ATTRFLAG_IMMUTABLE) < 0)
  log_err_exit ("attr_set_flags session-id");
 
+ // Setup profiling
+#if HAVE_CALIPER
+ cali_begin_string_byname ("flux.type", "main");
+ cali_begin_int_byname ("flux.tid", syscall(SYS_gettid));
+ cali_begin_string_byname ("binary", argv[0]);
+ cali_begin_int_byname ("flux.rank", ctx.rank);
+#endif
+
  /* Create directory for sockets, and a subdirectory specific
  * to this rank that will contain the pidfile and local connector socket.
  * (These may have already been called by boot method)

src/broker/module.c

@@ -42,6 +42,10 @@
 #include <argz.h>
 #include <czmq.h>
 #include <flux/core.h>
+#if HAVE_CALIPER
+#include <caliper/cali.h>
+#include <sys/syscall.h>
+#endif
 
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/xzmalloc.h"
@@ -51,6 +55,7 @@
 #include "module.h"
 #include "modservice.h"
 
+
 #define MODULE_MAGIC 0xfeefbe01
 struct module_struct {
  int magic;
@@ -114,6 +119,13 @@ static void *module_thread (void *arg)
 
  assert (p->zctx);
 
+#if HAVE_CALIPER
+ cali_begin_string_byname ("flux.type", "module");
+ cali_begin_int_byname ("flux.tid", syscall (SYS_gettid));
+ cali_begin_int_byname ("flux.rank", p->rank);
+ cali_begin_string_byname ("flux.name", p->name);
+#endif
+
  /* Connect to broker socket, enable logging, register built-in services
  */
  if (!(p->h = flux_open (uri, 0)))

src/common/Makefile.am

@@ -18,7 +18,7 @@ libflux_internal_la_LIBADD = \
  $(builddir)/libsubprocess/libsubprocess.la \
  $(builddir)/libcompat/libcompat.la \
  $(LIBMUNGE) $(JSON_LIBS) $(ZMQ_LIBS) $(LIBPTHREAD) $(LIBUTIL) \
- $(LIBDL) $(LIBRT)
+ $(LIBDL) $(LIBRT) $(LIBCALIPER)
 libflux_internal_la_LDFLAGS = $(san_ld_zdef_flag)
 
 lib_LTLIBRARIES = libflux-core.la libflux-optparse.la

src/common/libflux/dispatch.c

@@ -26,18 +26,22 @@
 #include "config.h"
 #endif
 #include <czmq.h>
+#if HAVE_CALIPER
+#include <caliper/cali.h>
+#include <sys/syscall.h>
+#endif
 
 #include "message.h"
 #include "reactor.h"
 #include "dispatch.h"
 #include "response.h"
+#include "info.h"
 #include "flog.h"
 
 #include "src/common/libutil/log.h"
 #include "src/common/libutil/coproc.h"
 #include "src/common/libutil/iterators.h"
 
-
 /* Fastpath for RPCs:
  * fastpath array translates response matchtags to message handlers,
  * bypassing the handlers zlist. Since the matchtag pools are LIFO,
@@ -51,6 +55,7 @@ struct fastpath {
  int len;
 };
 
+
 struct dispatch {
  flux_t h;
  zlist_t *handlers;
@@ -61,6 +66,10 @@ struct dispatch {
  flux_watcher_t *w;
  int running_count;
  int usecount;
+#if HAVE_CALIPER
+ cali_id_t prof_msg_type;
+ cali_id_t prof_msg_topic;
+#endif
 };
 
 #define HANDLER_MAGIC 0x44433322
@@ -144,7 +153,14 @@ static struct dispatch *dispatch_get (flux_t h)
  goto nomem;
  fastpath_init (&d->norm);
  fastpath_init (&d->group);
-
+#if HAVE_CALIPER
+ d->prof_msg_type = cali_create_attribute("flux.message.type",
+ CALI_TYPE_STRING,
+ CALI_ATTR_SKIP_EVENTS);
+ d->prof_msg_topic = cali_create_attribute("flux.message.topic",
+ CALI_TYPE_STRING,
+ CALI_ATTR_DEFAULT);
+#endif
  flux_aux_set (h, "flux::dispatch", d, dispatch_destroy);
  }
  return d;
@@ -556,15 +572,30 @@ static void handle_cb (flux_reactor_t *r, flux_watcher_t *hw,
  rc = 0; /* ignore mangled message */
  goto done;
  }
+
+ const char * topic;
+ flux_msg_get_topic(msg, &topic);
  /* Add any new handlers here, making handler creation
  * safe to call during handlers list traversal below.
  */
  if (transfer_items_zlist (d->handlers_new, d->handlers) < 0)
  goto done;
+
+#if defined(HAVE_CALIPER)
+ cali_begin_string(d->prof_msg_type, flux_msg_typestr(type));
+ cali_begin_string(d->prof_msg_topic, topic);
+#endif
+
  if ((flux_flags_get (d->h) & FLUX_O_COPROC))
  match = dispatch_message_coproc (d, msg, type);
  else
  match = dispatch_message (d, msg, type);
+
+#if defined(HAVE_CALIPER)
+ cali_end(d->prof_msg_topic);
+ cali_end(d->prof_msg_type);
+#endif
+
  if (match < 0)
  goto done;
  /* Destroy handlers here, making handler destruction

src/common/libutil/Makefile.am

@@ -93,7 +93,7 @@ test_ldadd = \
  $(top_builddir)/src/common/libtap/libtap.la \
  $(top_builddir)/src/common/liblsd/liblsd.la \
  $(top_builddir)/src/common/libev/libev.la \
- $(JSON_LIBS) $(ZMQ_LIBS) $(LIBPTHREAD) $(LIBRT)
+ $(JSON_LIBS) $(ZMQ_LIBS) $(LIBPTHREAD) $(LIBRT) $(LIBCALIPER)
 
 test_cppflags = \
  -I$(top_srcdir)/src/common/libtap \