Support sampling duration, sampling TIDs (#142)

- Sampling duration config values
  - OMNITRACE_SAMPLING_DURATION
  - OMNITRACE_PROCESS_SAMPLING_DURATION
  - Disables sampling after this time (in seconds) has elapsed 
- Sampling thread-id config values
  - OMNITRACE_SAMPLING_TIDS
  - OMNITRACE_SAMPLING_CPUTIME_TIDS
  - OMNITRACE_SAMPLING_REALTIME_TIDS
  - Allows user to select certain threads for sampling
- Miscellaneous
  - Tweaked the finalization verbosity messages
  - moved sampling-on-child-threads into runtime.hpp and runtime.cpp
  - fixed submodule dyninst header install

cmake/Packages.cmake

@@ -314,8 +314,7 @@ if(OMNITRACE_BUILD_DYNINST)
                 TARGETS ${_LIB}
                 DESTINATION ${CMAKE_INSTALL_LIBDIR}/omnitrace
                 COMPONENT dyninst
-                PUBLIC_HEADER DESTINATION ${PROJECT_BINARY_DIR}/.discard/omnitrace/include
-                )
+                PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/omnitrace/dyninst)
         endif()
     endforeach()
 

source/lib/common/defines.h.in

@@ -97,7 +97,7 @@
 #define OMNITRACE_STRINGIZE(X)           OMNITRACE_STRINGIZE2(X)
 #define OMNITRACE_STRINGIZE2(X)          #X
 #define OMNITRACE_VAR_NAME_COMBINE(X, Y) X##Y
-#define OMNITRACE_VARIABLE(Y)            OMNITRACE_VAR_NAME_COMBINE(_omni_var_, Y)
+#define OMNITRACE_VARIABLE(X, Y)         OMNITRACE_VAR_NAME_COMBINE(X, Y)
 #define OMNITRACE_LINESTR                OMNITRACE_STRINGIZE(__LINE__)
 #define OMNITRACE_ESC(...)               __VA_ARGS__
 

source/lib/omnitrace/library.cpp

@@ -337,27 +337,26 @@ omnitrace_init_tooling_hidden()
         if(get_state() > State::Active) return;
         if(get_use_process_sampling())
         {
-            pthread_gotcha::push_enable_sampling_on_child_threads(false);
+            OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
             process_sampler::setup();
-            pthread_gotcha::pop_enable_sampling_on_child_threads();
         }
         if(get_use_sampling())
         {
-            pthread_gotcha::push_enable_sampling_on_child_threads(false);
+            OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
             sampling::setup();
-            pthread_gotcha::pop_enable_sampling_on_child_threads();
-            pthread_gotcha::push_enable_sampling_on_child_threads(get_use_sampling());
+        }
+        if(get_use_sampling())
+        {
+            push_enable_sampling_on_child_threads(get_use_sampling());
             sampling::unblock_signals();
         }
         get_main_bundle()->start();
         set_state(State::Active);  // set to active as very last operation
     } };
 
-    if(get_use_sampling())
-    {
-        pthread_gotcha::push_enable_sampling_on_child_threads(false);
-        sampling::block_signals();
-    }
+    OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
+
+    if(get_use_sampling()) sampling::block_signals();
 
     if(get_use_critical_trace())
     {
@@ -426,8 +425,8 @@ omnitrace_init_tooling_hidden()
 
         for(const auto& itr : _disabled_categories)
         {
-            OMNITRACE_VERBOSE(1, "Disabling perfetto track event category: %s\n",
-                              itr.c_str());
+            OMNITRACE_VERBOSE_F(1, "Disabling perfetto track event category: %s\n",
+                                itr.c_str());
             track_event_cfg.add_disabled_categories(itr);
         }
 
@@ -581,6 +580,8 @@ omnitrace_finalize_hidden(void)
         return;
     }
 
+    if(get_verbose() >= 0 || get_debug()) fprintf(stderr, "\n");
+
     OMNITRACE_VERBOSE_F(0, "finalizing...\n");
     thread_info::set_stop(comp::wall_clock::record());
 
@@ -604,8 +605,8 @@ omnitrace_finalize_hidden(void)
 
     set_state(State::Finalized);
 
-    pthread_gotcha::push_enable_sampling_on_child_threads(false);
-    pthread_gotcha::set_sampling_on_all_future_threads(false);
+    push_enable_sampling_on_child_threads(false);
+    set_sampling_on_all_future_threads(false);
 
     auto _debug_init  = get_debug_finalize();
     auto _debug_value = get_debug();
@@ -614,8 +615,6 @@ omnitrace_finalize_hidden(void)
         if(_debug_init) config::set_setting_value("OMNITRACE_DEBUG", _debug_value);
     } };
 
-    OMNITRACE_DEBUG_F("\n");
-
     auto& _thread_bundle = thread_data<omnitrace_thread_bundle_t>::instance();
     if(_thread_bundle) _thread_bundle->stop();
 
@@ -713,7 +712,7 @@ omnitrace_finalize_hidden(void)
         comp::roctracer::shutdown();
 
         // join extra thread(s) used by roctracer
-        OMNITRACE_VERBOSE_F(1, "Waiting on roctracer tasks...\n");
+        OMNITRACE_VERBOSE_F(2, "Waiting on roctracer tasks...\n");
         tasking::join();
     }
 
@@ -734,10 +733,11 @@ omnitrace_finalize_hidden(void)
     // report the high-level metrics for the process
     if(get_main_bundle())
     {
+        if(get_verbose() >= 0 || get_debug()) fprintf(stderr, "\n");
         std::string _msg = JOIN("", *get_main_bundle());
         auto        _pos = _msg.find(">>>  ");
         if(_pos != std::string::npos) _msg = _msg.substr(_pos + 5);
-        OMNITRACE_PRINT("%s\n", _msg.c_str());
+        OMNITRACE_VERBOSE_F(0, "%s\n", _msg.c_str());
         OMNITRACE_DEBUG_F("Resetting main bundle...\n");
         get_main_bundle()->reset();
     }
@@ -754,10 +754,12 @@ omnitrace_finalize_hidden(void)
             std::string _msg = JOIN("", *itr);
             auto        _pos = _msg.find(">>>  ");
             if(_pos != std::string::npos) _msg = _msg.substr(_pos + 5);
-            OMNITRACE_VERBOSE(0, "%s\n", _msg.c_str());
+            OMNITRACE_VERBOSE_F(0, "%s\n", _msg.c_str());
         }
     }
 
+    if(get_verbose() >= 0 || get_debug()) fprintf(stderr, "\n");
+
     // ensure that all the MT instances are flushed
     if(get_use_sampling())
     {
@@ -813,6 +815,16 @@ omnitrace_finalize_hidden(void)
         tasking::join();
     }
 
+    // shutdown tasking before timemory is finalized, especially the roctracer thread-pool
+    OMNITRACE_VERBOSE_F(1, "Shutting down thread-pools...\n");
+    tasking::shutdown();
+
+    if(get_use_code_coverage())
+    {
+        OMNITRACE_VERBOSE_F(1, "Post-processing the code coverage...\n");
+        coverage::post_process();
+    }
+
     bool _perfetto_output_error = false;
     if(get_use_perfetto() && !is_system_backend())
     {
@@ -821,11 +833,7 @@ omnitrace_finalize_hidden(void)
         OMNITRACE_CI_THROW(tracing_session == nullptr,
                            "Null pointer to the tracing session");
 
-        if(get_verbose() >= 0) fprintf(stderr, "\n");
-        if(get_verbose() >= 0 || get_debug())
-            fprintf(stderr, "%s[%s][%s]|%i> Flushing perfetto...%s\n",
-                    tim::log::color::info(), TIMEMORY_PROJECT_NAME, OMNITRACE_FUNCTION,
-                    dmp::rank(), tim::log::color::end());
+        OMNITRACE_VERBOSE_F(0, "Finalizing perfetto...\n");
 
         // Make sure the last event is closed for this example.
         perfetto::TrackEvent::Flush();
@@ -905,16 +913,6 @@ omnitrace_finalize_hidden(void)
         }
     }
 
-    // shutdown tasking before timemory is finalized, especially the roctracer thread-pool
-    OMNITRACE_VERBOSE_F(1, "Shutting down thread-pools...\n");
-    tasking::shutdown();
-
-    OMNITRACE_VERBOSE_F(1, "Shutting down thread-pools...\n");
-    if(get_use_code_coverage())
-    {
-        coverage::post_process();
-    }
-
     tim::manager::instance()->add_metadata([](auto& ar) {
         auto _maps = tim::procfs::read_maps(process::get_id());
         auto _libs = std::set<std::string>{};

source/lib/omnitrace/library/components/pthread_create_gotcha.cpp

@@ -22,7 +22,6 @@
 
 #include "library/components/pthread_create_gotcha.hpp"
 #include "library/components/category_region.hpp"
-#include "library/components/pthread_gotcha.hpp"
 #include "library/components/roctracer.hpp"
 #include "library/config.hpp"
 #include "library/debug.hpp"
@@ -213,9 +212,8 @@ pthread_create_gotcha::wrapper::operator()() const
         if(m_enable_sampling)
         {
             _is_sampling = true;
-            pthread_gotcha::push_enable_sampling_on_child_threads(false);
+            OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
             _signals = sampling::setup();
-            pthread_gotcha::pop_enable_sampling_on_child_threads();
             sampling::unblock_signals();
         }
     }
@@ -336,7 +334,7 @@ pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
     auto        _active       = (get_state() == ::omnitrace::State::Active && !_disabled);
     auto        _coverage     = (get_mode() == Mode::Coverage);
     auto        _use_sampling = get_use_sampling();
-    auto        _sample_child = pthread_gotcha::sampling_enabled_on_child_threads();
+    auto        _sample_child = sampling_enabled_on_child_threads();
     auto        _tid          = utility::get_thread_index();
     auto        _use_bundle   = (_active && !_coverage);
     const auto& _info = thread_info::init(!_active || !_sample_child || _disabled);

source/lib/omnitrace/library/components/pthread_gotcha.cpp

@@ -69,14 +69,6 @@ namespace
 using bundle_t = tim::lightweight_tuple<component::pthread_create_gotcha_t,
                                         component::pthread_mutex_gotcha_t>;
 
-auto&
-get_sampling_on_child_threads_history(int64_t _idx = utility::get_thread_index())
-{
-    static auto _v = utility::get_filled_array<OMNITRACE_MAX_THREADS>(
-        []() { return utility::get_reserved_vector<bool>(32); });
-    return _v.at(_idx);
-}
-
 auto&
 get_bundle()
 {
@@ -112,51 +104,6 @@ pthread_gotcha::shutdown()
     }
 }
 
-bool
-pthread_gotcha::sampling_enabled_on_child_threads()
-{
-    return sampling_on_child_threads();
-}
-
-bool
-pthread_gotcha::push_enable_sampling_on_child_threads(bool _v)
-{
-    bool _last                  = sampling_on_child_threads();
-    sampling_on_child_threads() = _v;
-    auto& _hist                 = get_sampling_on_child_threads_history();
-    _hist.emplace_back(_last);
-    return _last;
-}
-
-bool
-pthread_gotcha::pop_enable_sampling_on_child_threads()
-{
-    auto& _hist = get_sampling_on_child_threads_history();
-    if(!_hist.empty())
-    {
-        bool _restored = _hist.back();
-        _hist.pop_back();
-        sampling_on_child_threads() = _restored;
-    }
-    return sampling_on_child_threads();
-}
-
-void
-pthread_gotcha::set_sampling_on_all_future_threads(bool _v)
-{
-    for(size_t i = 0; i < max_supported_threads; ++i)
-        get_sampling_on_child_threads_history(i).emplace_back(_v);
-}
-
-bool&
-pthread_gotcha::sampling_on_child_threads()
-{
-    static thread_local bool _v = get_sampling_on_child_threads_history().empty()
-                                      ? false
-                                      : get_sampling_on_child_threads_history().back();
-    return _v;
-}
-
 void
 pthread_gotcha::start()
 {

source/lib/omnitrace/library/components/pthread_gotcha.hpp

@@ -42,22 +42,7 @@ struct pthread_gotcha : tim::component::base<pthread_gotcha, void>
     static void configure();
     static void shutdown();
 
-    // query current value
-    static bool sampling_enabled_on_child_threads();
-
-    // use this to disable sampling in a region (e.g. right before thread creation)
-    static bool push_enable_sampling_on_child_threads(bool _v);
-
-    // use this to restore previous setting
-    static bool pop_enable_sampling_on_child_threads();
-
-    // make sure every newly created thead starts with this value
-    static void set_sampling_on_all_future_threads(bool _v);
-
     static void start();
     static void stop();
-
-private:
-    static bool& sampling_on_child_threads();
 };
 }  // namespace omnitrace

source/lib/omnitrace/library/components/pthread_mutex_gotcha.cpp

@@ -22,7 +22,6 @@
 
 #include "library/components/pthread_mutex_gotcha.hpp"
 #include "library/components/category_region.hpp"
-#include "library/components/pthread_gotcha.hpp"
 #include "library/config.hpp"
 #include "library/critical_trace.hpp"
 #include "library/debug.hpp"
@@ -293,7 +292,7 @@ pthread_mutex_gotcha::is_disabled()
 {
     return (get_state() != ::omnitrace::State::Active ||
             get_thread_state() != ThreadState::Enabled ||
-            (get_use_sampling() && !pthread_gotcha::sampling_enabled_on_child_threads()));
+            (get_use_sampling() && !sampling_enabled_on_child_threads()));
 }
 }  // namespace component
 }  // namespace omnitrace

source/lib/omnitrace/library/components/roctracer.cpp

@@ -22,13 +22,13 @@
 
 #include "library/components/roctracer.hpp"
 #include "library/common.hpp"
-#include "library/components/pthread_gotcha.hpp"
 #include "library/config.hpp"
 #include "library/debug.hpp"
 #include "library/defines.hpp"
 #include "library/dynamic_library.hpp"
 #include "library/redirect.hpp"
 #include "library/roctracer.hpp"
+#include "library/runtime.hpp"
 #include "library/sampling.hpp"
 #include "library/thread_data.hpp"
 
@@ -121,7 +121,7 @@ roctracer::setup()
     roctracer_is_setup() = true;
 
     OMNITRACE_VERBOSE_F(1, "setting up roctracer...\n");
-    pthread_gotcha::push_enable_sampling_on_child_threads(false);
+    OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
 
     dynamic_library _amdhip64{ "OMNITRACE_ROCTRACER_LIBAMDHIP64",
                                find_library_path("libamdhip64.so",
@@ -169,8 +169,6 @@ roctracer::setup()
     for(auto& itr : roctracer_setup_routines())
         itr.second();
 
-    pthread_gotcha::pop_enable_sampling_on_child_threads();
-
     OMNITRACE_VERBOSE_F(1, "roctracer is setup\n");
 }