#1983: runnable: optimize objgroup dispatch

DARMA-tasking · Mar 30, 2023 · 2563d13 · 2563d13
1 parent a642970
commit 2563d13
Show file tree

Hide file tree

Showing 12 changed files with 166 additions and 342 deletions.
diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc
@@ -495,10 +495,16 @@ EventType ActiveMessenger::doMessageSend(
     } else {
       recordLBDataCommForSend(dest, base, base.size());
 
-      runnable::makeRunnable(base, true, envelopeGetHandler(msg->env), dest)
-        .withTDEpochFromMsg(is_term)
-        .withLBData(&bare_handler_lb_data_, bare_handler_dummy_elm_id_for_lb_data_)
-        .enqueue();
+      auto han = envelopeGetHandler(msg->env);
+      bool const is_obj = HandlerManagerType::isHandlerObjGroup(han);
+      if (is_obj) {
+        objgroup::dispatchObjGroup(base, han, dest, nullptr);
+      } else {
+        runnable::makeRunnable(base, true, envelopeGetHandler(msg->env), dest)
+          .withTDEpochFromMsg(is_term)
+          .withLBData(&bare_handler_lb_data_, bare_handler_dummy_elm_id_for_lb_data_)
+          .enqueue();
+      }
     }
     return no_event;
   }
@@ -955,8 +961,7 @@ void ActiveMessenger::prepareActiveMsgToRun(
 
   bool const is_obj = HandlerManagerType::isHandlerObjGroup(handler);
   if (is_obj) {
-    vtAbortIf(cont != nullptr, "Must be nullptr");
-    objgroup::dispatchObjGroup(base, handler);
+    objgroup::dispatchObjGroup(base, handler, from_node, cont);
   } else {
     runnable::makeRunnable(base, not is_term, handler, from_node)
       .withContinuation(cont)

diff --git a/src/vt/objgroup/dispatch/dispatch.h b/src/vt/objgroup/dispatch/dispatch.h
diff --git a/src/vt/objgroup/dispatch/dispatch.impl.h b/src/vt/objgroup/dispatch/dispatch.impl.h
diff --git a/src/vt/objgroup/dispatch/dispatch_base.h b/src/vt/objgroup/dispatch/dispatch_base.h
diff --git a/src/vt/objgroup/manager.cc b/src/vt/objgroup/manager.cc
@@ -83,32 +83,6 @@ ObjGroupProxyType ObjGroupManager::getProxy(ObjGroupProxyType proxy) {
   return proxy;
 }
 
-void ObjGroupManager::dispatch(MsgSharedPtr<ShortMessage> msg, HandlerType han) {
-  // Extract the control-bit sequence from the handler
-  auto const ctrl = HandlerManager::getHandlerControl(han);
-  vt_debug_print(
-    verbose, objgroup,
-    "dispatch: ctrl={:x}, han={:x}\n", ctrl, han
-  );
-  auto const node = 0;
-  auto const proxy = proxy::ObjGroupProxy::create(ctrl, node, true);
-  auto dispatch_iter = dispatch_.find(proxy);
-  vt_debug_print(
-    normal, objgroup,
-    "dispatch: try ctrl={:x}, han={:x}, has dispatch={}\n",
-    ctrl, han, dispatch_iter != dispatch_.end()
-  );
-  if (dispatch_iter == dispatch_.end()) {
-    auto const epoch = envelopeGetEpoch(msg->env);
-    if (epoch != no_epoch and epoch != term::any_epoch_sentinel) {
-      theTerm()->produce(epoch);
-    }
-    pending_[proxy].push_back(msg);
-  } else {
-    dispatch_iter->second->run(han,msg.get());
-  }
-}
-
 ObjGroupProxyType ObjGroupManager::makeCollectiveImpl(
   std::string const& label, HolderBasePtrType base, void* obj_ptr
 ) {
@@ -163,12 +137,12 @@ elm::ElementIDStruct ObjGroupManager::getNextElm(ObjGroupProxyType proxy) {
   }
 }
 
-void dispatchObjGroup(MsgSharedPtr<ShortMessage> msg, HandlerType han) {
-  vt_debug_print(
-    verbose, objgroup,
-    "dispatchObjGroup: han={:x}\n", han
-  );
-  return theObjGroup()->dispatch(msg,han);
+std::unordered_map<ObjGroupProxyType, std::unique_ptr<holder::HolderBase>>& getObjs() {
+  return theObjGroup()->objs_;
+}
+
+std::unordered_map<ObjGroupProxyType, std::vector<ActionType>>& getPending() {
+  return theObjGroup()->pending_;
 }
 
 }} /* end namespace vt::objgroup */
diff --git a/src/vt/objgroup/manager.fwd.h b/src/vt/objgroup/manager.fwd.h
@@ -60,7 +60,13 @@ namespace detail {
 holder::HolderBase* getHolderBase(HandlerType handler);
 } /* end namespace detail */
 
-void dispatchObjGroup(MsgSharedPtr<ShortMessage> msg, HandlerType han);
+template <typename MsgT>
+void dispatchObjGroup(
+  MsgSharedPtr<MsgT> msg, HandlerType han, NodeType from_node, ActionType cont
+);
+
+std::unordered_map<ObjGroupProxyType, std::unique_ptr<holder::HolderBase>>& getObjs();
+std::unordered_map<ObjGroupProxyType, std::vector<ActionType>>& getPending();
 
 template <typename MsgT>
 messaging::PendingSend send(MsgSharedPtr<MsgT> msg, HandlerType han, NodeType node);