diff --git a/configure.ac b/configure.ac index 987b503df551..55d9cbffa609 100644 --- a/configure.ac +++ b/configure.ac @@ -69,6 +69,17 @@ X_AC_CHECK_COND_LIB(dl, dlerror) X_AC_MALLOC AC_CHECK_LIB(m, floor) +LIBCALIPER= +AC_ARG_ENABLE(caliper, + [ --enable-caliper[=OPTS] Use caliper for profiling. [default=no] [OPTS=no/yes]], , + [enable_caliper="no"]) + +if test "$enable_caliper" = "yes"; then + AC_SUBST([LIBCALIPER], ["-lcaliper"]) + AC_DEFINE([HAVE_CALIPER], [1], [Define if you have libcaliper]) + AC_DEFINE([PROFILING], [1], [Compile in profiling hooks]) +fi + AC_MSG_CHECKING(--enable-python argument) AC_ARG_ENABLE(python, [ --enable-python[=OPTS] Include Python bindings. [default=yes] [OPTS=no/yes]], , diff --git a/src/broker/broker.c b/src/broker/broker.c index df50ab5dcc30..c57fb5118085 100644 --- a/src/broker/broker.c +++ b/src/broker/broker.c @@ -54,6 +54,10 @@ #error gperftools headers not configured #endif #endif /* WITH_TCMALLOC */ +#if HAVE_CALIPER +#include +#include +#endif #include "src/common/libutil/log.h" #include "src/common/libutil/xzmalloc.h" @@ -247,6 +251,7 @@ int main (int argc, char *argv[]) int security_set = 0; int e; + memset (&ctx, 0, sizeof (ctx)); log_init (argv[0]); @@ -431,6 +436,14 @@ int main (int argc, char *argv[]) if (attr_set_flags (ctx.attrs, "session-id", FLUX_ATTRFLAG_IMMUTABLE) < 0) log_err_exit ("attr_set_flags session-id"); + // Setup profiling +#if HAVE_CALIPER + cali_begin_string_byname ("flux.type", "main"); + cali_begin_int_byname ("flux.tid", syscall(SYS_gettid)); + cali_begin_string_byname ("binary", argv[0]); + cali_begin_int_byname ("flux.rank", ctx.rank); +#endif + /* Create directory for sockets, and a subdirectory specific * to this rank that will contain the pidfile and local connector socket. * (These may have already been called by boot method) diff --git a/src/broker/module.c b/src/broker/module.c index 52b1874433a3..108ef83741b1 100644 --- a/src/broker/module.c +++ b/src/broker/module.c @@ -42,6 +42,10 @@ #include #include #include +#if HAVE_CALIPER +#include +#include +#endif #include "src/common/libutil/log.h" #include "src/common/libutil/xzmalloc.h" @@ -51,6 +55,7 @@ #include "module.h" #include "modservice.h" + #define MODULE_MAGIC 0xfeefbe01 struct module_struct { int magic; @@ -114,6 +119,13 @@ static void *module_thread (void *arg) assert (p->zctx); +#if HAVE_CALIPER + cali_begin_string_byname ("flux.type", "module"); + cali_begin_int_byname ("flux.tid", syscall (SYS_gettid)); + cali_begin_int_byname ("flux.rank", p->rank); + cali_begin_string_byname ("flux.name", p->name); +#endif + /* Connect to broker socket, enable logging, register built-in services */ if (!(p->h = flux_open (uri, 0))) diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 86b88f5f0ce8..be466d9415b0 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -18,7 +18,7 @@ libflux_internal_la_LIBADD = \ $(builddir)/libsubprocess/libsubprocess.la \ $(builddir)/libcompat/libcompat.la \ $(LIBMUNGE) $(JSON_LIBS) $(ZMQ_LIBS) $(LIBPTHREAD) $(LIBUTIL) \ - $(LIBDL) $(LIBRT) + $(LIBDL) $(LIBRT) $(LIBCALIPER) libflux_internal_la_LDFLAGS = $(san_ld_zdef_flag) lib_LTLIBRARIES = libflux-core.la libflux-optparse.la diff --git a/src/common/libflux/dispatch.c b/src/common/libflux/dispatch.c index 62d2994e9fe8..10d793e0c0f8 100644 --- a/src/common/libflux/dispatch.c +++ b/src/common/libflux/dispatch.c @@ -26,18 +26,22 @@ #include "config.h" #endif #include +#if HAVE_CALIPER +#include +#include +#endif #include "message.h" #include "reactor.h" #include "dispatch.h" #include "response.h" +#include "info.h" #include "flog.h" #include "src/common/libutil/log.h" #include "src/common/libutil/coproc.h" #include "src/common/libutil/iterators.h" - /* Fastpath for RPCs: * fastpath array translates response matchtags to message handlers, * bypassing the handlers zlist. Since the matchtag pools are LIFO, @@ -51,6 +55,7 @@ struct fastpath { int len; }; + struct dispatch { flux_t h; zlist_t *handlers; @@ -61,6 +66,10 @@ struct dispatch { flux_watcher_t *w; int running_count; int usecount; +#if HAVE_CALIPER + cali_id_t prof_msg_type; + cali_id_t prof_msg_topic; +#endif }; #define HANDLER_MAGIC 0x44433322 @@ -144,7 +153,14 @@ static struct dispatch *dispatch_get (flux_t h) goto nomem; fastpath_init (&d->norm); fastpath_init (&d->group); - +#if HAVE_CALIPER + d->prof_msg_type = cali_create_attribute("flux.message.type", + CALI_TYPE_STRING, + CALI_ATTR_SKIP_EVENTS); + d->prof_msg_topic = cali_create_attribute("flux.message.topic", + CALI_TYPE_STRING, + CALI_ATTR_DEFAULT); +#endif flux_aux_set (h, "flux::dispatch", d, dispatch_destroy); } return d; @@ -556,15 +572,30 @@ static void handle_cb (flux_reactor_t *r, flux_watcher_t *hw, rc = 0; /* ignore mangled message */ goto done; } + + const char * topic; + flux_msg_get_topic(msg, &topic); /* Add any new handlers here, making handler creation * safe to call during handlers list traversal below. */ if (transfer_items_zlist (d->handlers_new, d->handlers) < 0) goto done; + +#if defined(HAVE_CALIPER) + cali_begin_string(d->prof_msg_type, flux_msg_typestr(type)); + cali_begin_string(d->prof_msg_topic, topic); +#endif + if ((flux_flags_get (d->h) & FLUX_O_COPROC)) match = dispatch_message_coproc (d, msg, type); else match = dispatch_message (d, msg, type); + +#if defined(HAVE_CALIPER) + cali_end(d->prof_msg_topic); + cali_end(d->prof_msg_type); +#endif + if (match < 0) goto done; /* Destroy handlers here, making handler destruction diff --git a/src/common/libutil/Makefile.am b/src/common/libutil/Makefile.am index 41995b4d7c24..7f93117fcc98 100644 --- a/src/common/libutil/Makefile.am +++ b/src/common/libutil/Makefile.am @@ -93,7 +93,7 @@ test_ldadd = \ $(top_builddir)/src/common/libtap/libtap.la \ $(top_builddir)/src/common/liblsd/liblsd.la \ $(top_builddir)/src/common/libev/libev.la \ - $(JSON_LIBS) $(ZMQ_LIBS) $(LIBPTHREAD) $(LIBRT) + $(JSON_LIBS) $(ZMQ_LIBS) $(LIBPTHREAD) $(LIBRT) $(LIBCALIPER) test_cppflags = \ -I$(top_srcdir)/src/common/libtap \