(DO NOT MERGE) - Tpetra: performance improvements to CrsMatrix::copyAndPermute - release candidate

packages/panzer/disc-fe/src/Panzer_AssemblyEngine_impl.hpp

@@ -15,7 +15,16 @@
 #include "Panzer_FieldManagerBuilder.hpp"
 #include "Panzer_AssemblyEngine_InArgs.hpp"
 #include "Panzer_GlobalEvaluationDataContainer.hpp"
+#include "Teuchos_Time.hpp"
 #include <sstream>
+#include <unordered_map>
+#include <vector>
+
+#define EXP_INCLUDED_FROM_PANXER_MINI_EM 0
+#if EXP_INCLUDED_FROM_PANXER_MINI_EM
+extern std::unordered_map<std::string, std::pair<double, std::vector<double>> >& Timers;
+extern bool in_eval_J;
+#endif
 
 //===========================================================================
 //===========================================================================
@@ -87,18 +96,30 @@ evaluate(const panzer::AssemblyEngineInArgs& in, const EvaluationFlags flags)
   }
 
   if ( flags.getValue() & EvaluationFlags::Scatter) {
+    [[maybe_unused]] double time = Teuchos::Time::wallTime();
     PANZER_FUNC_TIME_MONITOR_DIFF("panzer::AssemblyEngine::evaluate_scatter("+PHX::print<EvalT>()+")",eval_scatter);
     {
+      [[maybe_unused]] double time1 = Teuchos::Time::wallTime();
       PANZER_FUNC_TIME_MONITOR_DIFF("panzer::AssemblyEngine::lof->ghostToGlobalContainer("+PHX::print<EvalT>()+")",lof_gtgc);
       m_lin_obj_factory->ghostToGlobalContainer(*in.ghostedContainer_,*in.container_,LOC::F | LOC::Mat);
+#if EXP_INCLUDED_FROM_PANXER_MINI_EM
+      if (in_eval_J) Timers["lof-g2gc"].first += -time1 + Teuchos::Time::wallTime();
+#endif
     }
     {
+      [[maybe_unused]] double time1 = Teuchos::Time::wallTime();
       PANZER_FUNC_TIME_MONITOR_DIFF("panzer::AssemblyEngine::gedc.ghostToGlobal("+PHX::print<EvalT>()+")",gedc_gtg);
       m_lin_obj_factory->beginFill(*in.container_);
       gedc.ghostToGlobal(LOC::F | LOC::Mat);
       m_lin_obj_factory->endFill(*in.container_);
+#if EXP_INCLUDED_FROM_PANXER_MINI_EM
+      if (in_eval_J) Timers["gedc-g2g"].first += -time1 + Teuchos::Time::wallTime();
+#endif
     }
     m_lin_obj_factory->endFill(*in.ghostedContainer_);
+#if EXP_INCLUDED_FROM_PANXER_MINI_EM
+    if (in_eval_J) Timers["eval_scatter"].first += -time + Teuchos::Time::wallTime();
+#endif
   }
 
   return;

packages/panzer/disc-fe/src/Panzer_ModelEvaluator_impl.hpp

@@ -47,6 +47,16 @@
 #include "Thyra_TpetraLinearOp.hpp"
 #include "Tpetra_CrsMatrix.hpp"
 
+
+#define EXP_INCLUDED_FROM_PANXER_MINI_EM 0
+#if EXP_INCLUDED_FROM_PANXER_MINI_EM
+extern bool use_eval_J;
+extern bool in_eval_J;
+extern std::unordered_map<std::string, std::pair<double, std::vector<double>> >& Timers;
+extern double timer_evalJ;
+extern double timer_capsg;
+#endif
+
 // Constructors/Initializers/Accessors
 
 template<typename Scalar>
@@ -1569,7 +1579,10 @@ evalModelImpl_basic(const Thyra::ModelEvaluatorBase::InArgs<Scalar> &inArgs,
   else if(Teuchos::is_null(f_out) && !Teuchos::is_null(W_out)) {
 
     PANZER_FUNC_TIME_MONITOR("panzer::ModelEvaluator::evalModel(J)");
-
+#if EXP_INCLUDED_FROM_PANXER_MINI_EM
+    double time_ = Teuchos::Time::wallTime();
+    in_eval_J = true;
+#endif
     // only add auxiliary global data if Jacobian is being formed
     ae_inargs.addGlobalEvaluationData(nonParamGlobalEvaluationData_);
 
@@ -1582,6 +1595,10 @@ evalModelImpl_basic(const Thyra::ModelEvaluatorBase::InArgs<Scalar> &inArgs,
     thGhostedContainer->initializeMatrix(0.0);
 
     ae_tm_.template getAsObject<panzer::Traits::Jacobian>()->evaluate(ae_inargs);
+#if EXP_INCLUDED_FROM_PANXER_MINI_EM
+    in_eval_J = !use_eval_J;
+    Timers["evalJ"].first += -time_ + Teuchos::Time::wallTime();
+#endif
   }
 
   // HACK: set A to null before calling responses to avoid touching the