#1983: runnable: overhaul to reduce allocations

src/vt/messaging/active.cc

@@ -955,11 +955,17 @@ void ActiveMessenger::prepareActiveMsgToRun(
     );
   }
 
-  runnable::makeRunnable(base, not is_term, handler, from_node)
-    .withContinuation(cont)
-    .withTDEpochFromMsg(is_term)
-    .withLBData(&bare_handler_lb_data_, bare_handler_dummy_elm_id_for_lb_data_)
-    .enqueue();
+  bool const is_obj = HandlerManagerType::isHandlerObjGroup(handler);
+  if (is_obj) {
+    vtAbortIf(cont != nullptr, "Must be nullptr");
+    objgroup::dispatchObjGroup(base, handler);
+  } else {
+    runnable::makeRunnable(base, not is_term, handler, from_node)
+      .withContinuation(cont)
+      .withTDEpochFromMsg(is_term)
+      .withLBData(&bare_handler_lb_data_, bare_handler_dummy_elm_id_for_lb_data_)
+      .enqueue();
+  }
 
   if (is_term) {
     tdRecvCount.increment(1);

src/vt/objgroup/dispatch/dispatch.impl.h

@@ -52,20 +52,15 @@ namespace vt { namespace objgroup { namespace dispatch {
 
 template <typename ObjT>
 void Dispatch<ObjT>::run(HandlerType han, BaseMessage* msg) {
-  using ActiveFnType = void(ObjT::*)(vt::BaseMessage*);
+  //using ActiveFnType = void(ObjT::*)(vt::BaseMessage*);
   vtAssert(obj_ != nullptr, "Must have a valid object");
-  // Consume if there is an epoch for this message
+
   auto tmsg = static_cast<vt::Message*>(msg);
-  auto cur_epoch = envelopeGetEpoch(tmsg->env);
-  if (cur_epoch != no_epoch) {
-    theMsg()->pushEpoch(cur_epoch);
-  }
-  auto base_func = auto_registry::getAutoHandlerObjGroup(han);
-  auto type_func = reinterpret_cast<ActiveFnType>(base_func);
-  (obj_->*type_func)(msg);
-  if (cur_epoch != no_epoch) {
-    theMsg()->popEpoch(cur_epoch);
-  }
+  auto m = promoteMsg(tmsg);
+  runnable::makeRunnable(m, true, han, theContext()->getNode())
+    .withObjGroup(obj_)
+    .withTDEpochFromMsg()
+    .enqueue();
 }
 
 }}} /* end namespace vt::objgroup::dispatch */

src/vt/registry/auto/auto_registry.h

@@ -62,7 +62,7 @@ namespace vt { namespace auto_registry {
 
 AutoActiveType const& getAutoHandler(HandlerType const handler);
 
-AutoActiveObjGroupType getAutoHandlerObjGroup(HandlerType han);
+AutoActiveObjGroupType const& getAutoHandlerObjGroup(HandlerType han);
 AutoHandlerType getAutoHandlerObjTypeIdx(HandlerType han);
 
 template <typename ObjT, typename MsgT, objgroup::ActiveObjType<MsgT, ObjT> f>

src/vt/registry/auto/auto_registry_common.h

@@ -241,14 +241,14 @@ using AutoActiveFunctorType       = BaseHandlersDispatcherPtr;
 using AutoActiveVCType            = BaseHandlersDispatcherPtr;
 using AutoActiveCollectionType    = BaseHandlersDispatcherPtr;
 using AutoActiveCollectionMemType = BaseHandlersDispatcherPtr;
+using AutoActiveObjGroupType      = BaseHandlersDispatcherPtr;
 using AutoActiveMapType           = BaseMapsDispatcherPtr;
 using AutoActiveMapFunctorType    = BaseMapsDispatcherPtr;
 
 using AutoActiveSeedMapType       = mapping::ActiveSeedMapFnPtrType;
 using AutoActiveRDMAGetType       = ActiveRDMAGetFnPtrType;
 using AutoActiveRDMAPutType       = ActiveRDMAPutFnPtrType;
 using AutoActiveIndexType         = std::size_t;
-using AutoActiveObjGroupType      = objgroup::ActiveObjAnyType;
 
 using HandlerManagerType = vt::HandlerManager;
 using AutoHandlerType = HandlerType;

src/vt/registry/auto/auto_registry_impl.h

@@ -54,7 +54,7 @@
 
 namespace vt { namespace auto_registry {
 
-inline AutoActiveObjGroupType getAutoHandlerObjGroup(HandlerType han) {
+inline AutoActiveObjGroupType const& getAutoHandlerObjGroup(HandlerType han) {
   using ContainerType = AutoActiveObjGroupContainerType;
 
   auto const id = HandlerManagerType::getHandlerIdentifier(han);
@@ -71,7 +71,7 @@ inline AutoHandlerType getAutoHandlerObjTypeIdx(HandlerType han) {
 template <typename ObjT, typename MsgT, objgroup::ActiveObjType<MsgT, ObjT> f>
 inline HandlerType makeAutoHandlerObjGroup(HandlerControlType ctrl) {
   using AdapterT =
-    FunctorAdapterMember<objgroup::ActiveObjType<MsgT, ObjT>, f, ObjT>;
+    FunctorAdapterMember<objgroup::ActiveObjType<MsgT, ObjT>, f, ObjT, MsgT>;
   using ContainerType = AutoActiveObjGroupContainerType;
   using RegInfoType = AutoRegInfoType<AutoActiveObjGroupType>;
   using FuncType = objgroup::ActiveObjAnyType;

src/vt/runnable/make_runnable.h

@@ -78,7 +78,6 @@ struct RunnableMaker {
   ) : impl_(in_impl),
       msg_(in_msg),
       handler_(in_handler),
-      is_void_(in_msg == nullptr),
       from_node_(in_from_node),
       has_msg_(in_msg != nullptr)
   { }
@@ -162,6 +161,20 @@ struct RunnableMaker {
     return std::move(*this);
   }
 
+  /**
+   * \brief Add an objgroup; sets up the handler
+   *
+   * \param[in] elm the collection element pointer
+   */
+  template <typename ElmT>
+  RunnableMaker&& withObjGroup(ElmT* elm) {
+    set_handler_ = true;
+    if (handler_ != uninitialized_handler) {
+      impl_->setupHandlerObjGroup(elm, handler_);
+    }
+    return std::move(*this);
+  }
+
   /**
    * \brief Add LB data for instrumentation
    *
@@ -261,28 +274,28 @@ struct RunnableMaker {
   }
 
   /**
-   * \brief Enqueue the runnable in the scheduler for execution later
+   * \brief Run the runnable immediately with a lambda
    */
-  void enqueue();
+  void runLambda(ActionType action) {
+    setup();
+    impl_->runLambda(action);
+    delete impl_;
+    impl_ = nullptr;
+    is_done_ = true;
+  }
 
   /**
-   * \brief Set an explicit task for this runnable (not going through normal
-   * handler)
-   *
-   * \param[in] task_ the task to execute
+   * \brief Enqueue the runnable in the scheduler for execution later
    */
-  RunnableMaker&& withExplicitTask(ActionType task_) {
-    impl_->setExplicitTask(task_);
-    return std::move(*this);
-  }
+  void enqueue();
 
 private:
   /**
    * \internal \brief Setup for running or enqueuing
    */
   void setup() {
     if (not set_handler_) {
-      impl_->setupHandler(handler_, is_void_);
+      impl_->setupHandler(handler_);
       set_handler_ = true;
     }
   }
@@ -292,7 +305,6 @@ struct RunnableMaker {
   MsgSharedPtr<MsgT> const& msg_;
   HandlerType handler_ = uninitialized_handler;
   bool set_handler_ = false;
-  bool is_void_ = false;
   NodeType from_node_ = uninitialized_destination;
   bool is_done_ = false;
   bool is_term_ = false;

src/vt/runnable/runnable.cc

@@ -53,83 +53,55 @@
 
 namespace vt { namespace runnable {
 
-void RunnableNew::setupHandler(HandlerType handler, bool is_void) {
+void RunnableNew::setupHandler(HandlerType handler) {
   using HandlerManagerType = HandlerManager;
-  bool const is_obj = HandlerManagerType::isHandlerObjGroup(handler);
 
-  if (not is_void) {
-    if (is_obj) {
-      task_ = [=] { objgroup::dispatchObjGroup(msg_, handler); };
-      return;
-    } else {
-      bool const is_auto = HandlerManagerType::isHandlerAuto(handler);
-      bool const is_functor = HandlerManagerType::isHandlerFunctor(handler);
-
-      if (is_auto && is_functor) {
-        auto const& func = auto_registry::getAutoHandlerFunctor(handler);
-        auto const num_args = auto_registry::getAutoHandlerFunctorArgs(handler);
-        if (num_args == 0) {
-          task_ = [=, &func] { func->dispatch(nullptr, nullptr); };
-        } else {
-          task_ = [=, &func] { func->dispatch(msg_.get(), nullptr); };
-        }
-
-        return;
-      } else {
-        bool const is_base_msg_derived =
-          HandlerManagerType::isHandlerBaseMsgDerived(handler);
-        if (is_base_msg_derived) {
-          auto const& func = auto_registry::getAutoHandler(handler);
-          task_ = [=, &func] { func->dispatch(msg_.get(), nullptr); };
-          return;
-        }
-
-        auto const& func = auto_registry::getScatterAutoHandler(handler);
-        task_ = [=, &func] { func->dispatch(msg_.get(), nullptr); };
-        return;
-      }
-    }
-  } else {
-    bool const is_auto = HandlerManagerType::isHandlerAuto(handler);
-    bool const is_functor = HandlerManagerType::isHandlerFunctor(handler);
+  bool const is_auto = HandlerManagerType::isHandlerAuto(handler);
+  bool const is_functor = HandlerManagerType::isHandlerFunctor(handler);
 
-    if (is_auto && is_functor) {
-      auto const& func = auto_registry::getAutoHandlerFunctor(handler);
-      task_ = [=, &func] { func->dispatch(nullptr, nullptr); };
-      return;
-    } else if (is_auto) {
-      bool const is_base_msg_derived =
-        HandlerManagerType::isHandlerBaseMsgDerived(handler);
-      if (is_base_msg_derived) {
-        auto const& func = auto_registry::getAutoHandler(handler);
-        task_ = [=, &func] { func->dispatch(msg_.get(), nullptr); };
-        return;
-      }
-
-      auto const& func = auto_registry::getScatterAutoHandler(handler);
-      task_ = [=, &func] { func->dispatch(msg_.get(), nullptr); };
+  if (is_auto && is_functor) {
+    f_.func_ = auto_registry::getAutoHandlerFunctor(handler).get();
+    return;
+  } else {
+    bool const is_base_msg_derived =
+      HandlerManagerType::isHandlerBaseMsgDerived(handler);
+    if (is_base_msg_derived) {
+      f_.func_ = auto_registry::getAutoHandler(handler).get();
       return;
-    } else {
-      vtAbort("Must be auto/functor for a void handler");
     }
+
+    is_scatter_ = true;
+    f_.func_scat_ = auto_registry::getScatterAutoHandler(handler).get();
+    return;
   }
 }
 
+void RunnableNew::setupHandlerObjGroup(void* obj, HandlerType handler) {
+  f_.func_ = auto_registry::getAutoHandlerObjGroup(handler).get();
+  obj_ = obj;
+}
+
 void RunnableNew::setupHandlerElement(
   vrt::collection::UntypedCollection* elm, HandlerType handler
 ) {
   auto const member = HandlerManager::isHandlerMember(handler);
-  auto const& func = member ?
-    auto_registry::getAutoHandlerCollectionMem(handler) :
-    auto_registry::getAutoHandlerCollection(handler);
-  task_ = [=, &func] { func->dispatch(msg_.get(), elm); };
+  f_.func_ = member ?
+    auto_registry::getAutoHandlerCollectionMem(handler).get() :
+    auto_registry::getAutoHandlerCollection(handler).get();
+  obj_ = elm;
 }
 
 void RunnableNew::setupHandlerElement(
   vrt::VirtualContext* elm, HandlerType handler
 ) {
-  auto const& func = auto_registry::getAutoHandlerVC(handler);
-  task_ = [=, &func] { func->dispatch(msg_.get(), elm); };
+  f_.func_ = auto_registry::getAutoHandlerVC(handler).get();
+  obj_ = elm;
+}
+
+void RunnableNew::runLambda(ActionType action) {
+  begin();
+  action();
+  end();
 }
 
 void RunnableNew::run() {
@@ -156,8 +128,6 @@ void RunnableNew::run() {
   begin();
 #endif
 
-  vtAssert(task_ != nullptr, "Must have a valid task to run");
-
 #if vt_check_enabled(fcontext)
   if (is_threaded_ and not theConfig()->vt_ult_disable) {
     auto tm = theSched()->getThreadManager();
@@ -167,7 +137,9 @@ void RunnableNew::run() {
       tm->getThread(tid_)->resume();
     } else {
       // allocate a new thread to run the task
-      tid_ = tm->allocateThreadRun(task_);
+      tid_ = tm->allocateThreadRun([&]{
+        f_.func_->dispatch(msg_ == nullptr ? nullptr : msg_.get(), obj_);
+      });
     }
 
     // check if it is done running, and save that state
@@ -185,7 +157,11 @@ void RunnableNew::run() {
     vt_force_use(is_threaded_, tid_)
 #endif
 
-    task_();
+    if (is_scatter_) {
+      f_.func_scat_->dispatch(msg_ == nullptr ? nullptr : msg_.get(), obj_);
+    } else {
+      f_.func_->dispatch(msg_ == nullptr ? nullptr : msg_.get(), obj_);
+    }
 
 #if vt_check_enabled(fcontext)
     done_ = true;

src/vt/runnable/runnable.h

@@ -96,6 +96,8 @@ struct Contexts {
 struct RunnableNew {
   template <typename... Args>
   using FnParamType = void(*)(Args...);
+  using DispatcherType = auto_registry::BaseHandlersDispatcherPtr::pointer;
+  using DispatcherScatterType = auto_registry::BaseScatterDispatcherPtr::pointer;
 
   /**
    * \brief Construct a new \c RunnableNew with a message
@@ -194,6 +196,14 @@ struct RunnableNew {
     vrt::collection::UntypedCollection* elm, HandlerType handler
   );
 
+  /**
+   * \brief Set up a handler to run on an object group
+   *
+   * \param[in] elm the object pointer
+   * \param[in] handler the handler ID bits
+   */
+  void setupHandlerObjGroup(void* obj, HandlerType handler);
+
   /**
    * \brief Set up a handler to run on an non-collection object
    *
@@ -206,9 +216,8 @@ struct RunnableNew {
    * \brief Set up a basic handler to run
    *
    * \param[in] handler the handler ID bits
-   * \param[in] is_void whether it's a void handler w/o an associated message
    */
-  void setupHandler(HandlerType handler, bool is_void = false);
+  void setupHandler(HandlerType handler);
 
   /**
    * \brief Run the task!
@@ -219,6 +228,11 @@ struct RunnableNew {
    */
   void run();
 
+  /**
+   * \brief Run the task as a lambda!
+   */
+  void runLambda(ActionType action);
+
 #if vt_check_enabled(fcontext)
   /**
    * \brief Get the thread ID associated with the runnable.
@@ -306,15 +320,6 @@ struct RunnableNew {
   bool isSuspended() const { return suspended_; }
 #endif
 
-  /**
-   * \brief Set an explicit task for the runnable bypassing the handler
-   *
-   * \param[in] task_in the task
-   */
-  void setExplicitTask(ActionType task_in) {
-    task_ = task_in;
-  }
-
   /**
    * \internal \brief Operator new for runnables targeting pool
    *
@@ -334,7 +339,12 @@ struct RunnableNew {
 private:
   detail::Contexts contexts_;               /**< The contexts  */
   MsgSharedPtr<BaseMsgType> msg_ = nullptr; /**< The associated message */
-  ActionType task_ = nullptr;               /**< The runnable's task  */
+  void* obj_ = nullptr;                     /**< Object pointer */
+  union {
+    DispatcherType func_;
+    DispatcherScatterType func_scat_;
+  } f_;
+  bool is_scatter_ = false;
 #if vt_check_enabled(fcontext)
   bool is_threaded_ = false;                /**< Whether ULTs are supported */
   bool done_ = false;                       /**< Whether task is complete */