repeated work in shared entity communication

[chakshinglee]

Currently, when generating traces, we create 3 SharedEntityCommunication objects to exchange Vector, DenseMatrix, and SparseMatrix. All of them take "edge to true edge" table as input, and generate an entity (edge in this case) to proc table entity_proc_ within the object.

smoothG/src/sharedentitycommunication.hpp

Lines 345 to 440 in 0eca47e

	entity_proc_ = new mfem::Table;
	int num_entities =
	entity_trueentity_->row_starts[1] - entity_trueentity_->row_starts[0];
	entity_master_ = new int[num_entities];
	entity_proc_->MakeI(num_entities);
	std::vector<std::pair<int, int> > trueentity_proc;
	for (int send = 0; send < comm_pkg->num_sends; ++send)
	{
	int proc = comm_pkg->send_procs[send];
	for (int j = comm_pkg->send_map_starts[send];
	j < comm_pkg->send_map_starts[send + 1];
	++j)
	{
	int trueentity = comm_pkg->send_map_elmts[j];
	trueentity_proc.push_back(std::make_pair(trueentity, proc));
	}
	}
	for (int entity = 0; entity < num_entities; ++entity)
	{
	MFEM_ASSERT((ete_offd_I_[entity + 1] - ete_offd_I_[entity] == 1) ||
	(ete_offd_I_[entity + 1] - ete_offd_I_[entity] == 0),
	"entity_trueentity has more than one column per row!");
	entity_master_[entity] = comm_rank_;
	entity_proc_->AddAColumnInRow(entity);
	if (ete_offd_I_[entity + 1] - ete_offd_I_[entity] == 0) // local
	{
	int trueentity = ete_diag_J_[ete_diag_I_[entity]];
	for (unsigned int i = 0; i < trueentity_proc.size(); ++i)
	{
	if (trueentity == trueentity_proc[i].first)
	{
	entity_proc_->AddAColumnInRow(entity);
	}
	}
	}
	if (ete_offd_I_[entity + 1] - ete_offd_I_[entity] == 1)
	{
	int col = ete_offd_J_[ete_offd_I_[entity]];
	for (int recv = 0; recv < comm_pkg->num_recvs; ++recv)
	{
	int proc = comm_pkg->recv_procs[recv];
	for (int k = comm_pkg->recv_vec_starts[recv];
	k < comm_pkg->recv_vec_starts[recv + 1];
	++k)
	{
	if (k == col)
	{
	// next line is really unnecessary, we never touch the
	// unowned rows of entity_proc_
	entity_proc_->AddAColumnInRow(entity);
	entity_master_[entity] =
	(proc < entity_master_[entity]) ? proc : entity_master_[entity];
	}
	}
	}
	}
	}
	entity_proc_->MakeJ();
	for (int entity = 0; entity < num_entities; ++entity)
	{
	MFEM_ASSERT((ete_offd_I_[entity + 1] - ete_offd_I_[entity] == 1) ||
	(ete_offd_I_[entity + 1] - ete_offd_I_[entity] == 0),
	"entity_trueentity has more than one column per row!");
	entity_proc_->AddConnection(entity, comm_rank_);
	if (ete_offd_I_[entity + 1] - ete_offd_I_[entity] == 0)
	{
	int trueentity = ete_diag_J_[ete_diag_I_[entity]];
	for (unsigned int i = 0; i < trueentity_proc.size(); ++i)
	{
	if (trueentity == trueentity_proc[i].first)
	{
	int proc = trueentity_proc[i].second;
	entity_proc_->AddConnection(entity, proc);
	}
	}
	}
	if (ete_offd_I_[entity + 1] - ete_offd_I_[entity] == 1)
	{
	int col = ete_offd_J_[ete_offd_I_[entity]];
	for (int recv = 0; recv < comm_pkg->num_recvs; ++recv)
	{
	int proc = comm_pkg->recv_procs[recv];
	for (int k = comm_pkg->recv_vec_starts[recv];
	k < comm_pkg->recv_vec_starts[recv + 1];
	++k)
	{
	if (k == col)
	{
	entity_proc_->AddConnection(entity, proc);
	}
	}
	}
	}
	}
	entity_proc_->ShiftUpI();
	entity_proc_->Finalize();

Is it right that entity_proc_ only needs to be generated once (i.e. it is independent of what type is being exchanged)? If so, can this class be modified to do that? One simple way may be to take the construction of entity_proc_ out of the class and take it as input instead of entity_trueentity. Otherwise, we can also split setup and the actual communication, and only template the communication part. Most likely the second approach will break template structure of the class.

If possible, it's always good to avoid redundant work. Nevertheless, I guess building entity_proc_ is not really that expensive, so if the modification is not easy to make, this issue can be ignored.