Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

STK: Snapshot 04-20-21 19:17 #9039

Merged
merged 1 commit into from
Apr 21, 2021
Merged

STK: Snapshot 04-20-21 19:17 #9039

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion packages/stk/stk_balance/stk_balance/internal/M2NDecomposer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,14 @@ M2NDecomposer::M2NDecomposer(stk::mesh::BulkData & bulkData,
{
}

unsigned
M2NDecomposer::num_required_subdomains_for_each_proc()
{
const unsigned numInitialSubdomains = m_bulkData.parallel_size();
const unsigned numFinalSubdomains = m_parsedOptions.targetNumProcs;
return (numFinalSubdomains / numInitialSubdomains) + (numFinalSubdomains % numInitialSubdomains > 0);
}

stk::mesh::EntityProcVec
M2NDecomposer::get_partition()
{
Expand All @@ -75,7 +83,6 @@ M2NDecomposer::map_new_subdomains_to_original_processors()
return targetSubdomainsToProc;
}


M2NDecomposerNested::M2NDecomposerNested(stk::mesh::BulkData & bulkData,
const stk::balance::BalanceSettings & balanceSettings,
const stk::balance::M2NParsedOptions & parsedOptions)
Expand Down
2 changes: 2 additions & 0 deletions packages/stk/stk_balance/stk_balance/internal/M2NDecomposer.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ class M2NDecomposer

virtual std::vector<unsigned> map_new_subdomains_to_original_processors();

unsigned num_required_subdomains_for_each_proc();

protected:
stk::mesh::BulkData & m_bulkData;
const stk::balance::BalanceSettings & m_balanceSettings;
Expand Down
81 changes: 32 additions & 49 deletions packages/stk/stk_balance/stk_balance/internal/MtoNRebalancer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,6 @@ namespace stk {
namespace balance {
namespace internal {

MtoNRebalancer::MtoNRebalancer(stk::mesh::BulkData &bulkData,
stk::mesh::Field<double> &targetField,
M2NDecomposer &decomposer,
const stk::balance::M2NParsedOptions &parsedOptions)
: m_bulkData(bulkData),
m_subdomainCreator(bulkData, parsedOptions.targetNumProcs),
m_decomposer(decomposer),
m_targetDecompField(targetField),
m_parsedOptions(parsedOptions)
{
}

MtoNRebalancer::MtoNRebalancer(stk::io::StkMeshIoBroker& ioBroker,
stk::mesh::Field<double> &targetField,
M2NDecomposer &decomposer,
Expand All @@ -52,43 +40,52 @@ MtoNRebalancer::map_new_subdomains_to_original_processors()
return m_decomposer.map_new_subdomains_to_original_processors();
}

void MtoNRebalancer::move_subdomains_such_that_entire_subdomain_doesnt_span_proc_boundaries(const std::vector<unsigned>& target_proc_to_starting_proc)
void MtoNRebalancer::move_final_subdomains_onto_this_processor(const std::vector<unsigned>& originalProcessorForEachFinalSubdomain)
{
store_off_target_proc_on_elements_before_moving_subdomains();
stk::balance::internal::rebalance(m_bulkData, target_proc_to_starting_proc, m_decomp);
move_entities_into_mapped_subdomain_parts(target_proc_to_starting_proc);
move_entities_into_mapped_subdomain_parts(originalProcessorForEachFinalSubdomain);
stk::balance::internal::rebalance(m_bulkData, originalProcessorForEachFinalSubdomain, m_decomp);
}

stk::io::EntitySharingInfo MtoNRebalancer::get_node_sharing_info(unsigned subdomain)
std::vector<unsigned>
MtoNRebalancer::get_final_subdomains_for_this_processor()
{
return m_subdomainCreator.get_node_sharing_info(subdomain);
const std::vector<unsigned> originalProcessorsForEachNewSubdomain = map_new_subdomains_to_original_processors();

std::vector<unsigned> finalSubdomainsForThisProc;
for (unsigned i = 0; i < originalProcessorsForEachNewSubdomain.size(); ++i) {
if (originalProcessorsForEachNewSubdomain[i] == static_cast<unsigned>(m_bulkData.parallel_rank())) {
finalSubdomainsForThisProc.push_back(i);
}
}

finalSubdomainsForThisProc.resize(m_decomposer.num_required_subdomains_for_each_proc(), SubdomainCreator::INVALID_SUBDOMAIN);

return finalSubdomainsForThisProc;
}

void MtoNRebalancer::move_entities_into_mapped_subdomain_parts(const std::vector<unsigned>& mappings)
{
m_subdomainCreator.declare_all_final_subdomain_parts();
m_bulkData.modification_begin();
change_parts_on_entities_on_all_subdomains(mappings);
m_bulkData.modification_end();
}
const stk::mesh::PartVector subdomainParts = m_subdomainCreator.declare_all_final_subdomain_parts();

void MtoNRebalancer::change_parts_on_entities_on_all_subdomains(const std::vector<unsigned>& subdomain_proc_mapping)
{
for(int i = 0; i < m_subdomainCreator.get_num_final_subdomains(); i++)
{
if(subdomain_proc_mapping[i] == static_cast<unsigned>(m_bulkData.parallel_rank()))
{
std::vector<stk::mesh::Entity> entities = get_entities_for_subdomain(i);
m_subdomainCreator.move_entities_into_final_subdomain_part(i, entities);
}
}
m_bulkData.modification_begin();
for (const stk::mesh::EntityProc & entityProc : m_decomp) {
m_bulkData.change_entity_parts(entityProc.first, stk::mesh::PartVector{subdomainParts[entityProc.second]});
}
m_bulkData.modification_end();
}

stk::mesh::BulkData& MtoNRebalancer::get_bulk()
{
return m_bulkData;
}

SubdomainCreator&
MtoNRebalancer::get_subdomain_creator()
{
return m_subdomainCreator;
}

std::vector<stk::mesh::Entity> MtoNRebalancer::get_entities_for_subdomain(size_t subdomain_num)
{
const stk::mesh::BucketVector &buckets = get_bulk().buckets(stk::topology::ELEMENT_RANK);
Expand Down Expand Up @@ -129,25 +126,11 @@ void MtoNRebalancer::store_off_target_proc_on_elements_before_moving_subdomains(
stk::balance::internal::put_entity_data_to_field(m_decomp, &m_targetDecompField);
}

bool MtoNRebalancer::does_this_proc_own_subdomain(unsigned subdomainOwner)
{
return (subdomainOwner == static_cast<unsigned>(get_bulk().parallel_rank()));
}

void MtoNRebalancer::create_subdomain_and_write(const std::string &filename,
unsigned subdomain,
int global_num_nodes,
int global_num_elems,
const stk::io::EntitySharingInfo &nodeSharingInfo,
void MtoNRebalancer::create_subdomain_and_write(const std::string &filename, unsigned subdomain,
int global_num_nodes, int global_num_elems,
int numSteps, double timeStep)
{
m_subdomainCreator.create_subdomain_and_write(filename,
subdomain,
global_num_nodes,
global_num_elems,
nodeSharingInfo,
numSteps,
timeStep);
m_subdomainCreator.create_subdomain_and_write(filename, subdomain, global_num_nodes, global_num_elems, numSteps, timeStep);
}

int MtoNRebalancer::get_num_target_subdomains()
Expand Down
21 changes: 6 additions & 15 deletions packages/stk/stk_balance/stk_balance/internal/MtoNRebalancer.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,6 @@ namespace internal {
class MtoNRebalancer
{
public:
MtoNRebalancer(stk::mesh::BulkData &bulkData,
stk::mesh::Field<double> &targetField,
M2NDecomposer &decomposer,
const stk::balance::M2NParsedOptions &num_target_procs);
MtoNRebalancer(stk::io::StkMeshIoBroker& ioBroker,
stk::mesh::Field<double> &targetField,
M2NDecomposer &decomposer,
Expand All @@ -67,26 +63,21 @@ class MtoNRebalancer

void decompose_mesh();
std::vector<unsigned> map_new_subdomains_to_original_processors();
std::vector<unsigned> get_final_subdomains_for_this_processor();

void move_subdomains_such_that_entire_subdomain_doesnt_span_proc_boundaries(const std::vector<unsigned>& target_proc_to_starting_proc);
stk::io::EntitySharingInfo get_node_sharing_info(unsigned subdomain);
void create_subdomain_and_write(const std::string &filename,
unsigned subdomain,
int global_num_nodes,
int global_num_elems,
const stk::io::EntitySharingInfo &nodeSharingInfo,
int numSteps = -1,
double timeStep = 0.0);
bool does_this_proc_own_subdomain(unsigned subdomainOwner);
void move_final_subdomains_onto_this_processor(const std::vector<unsigned>& finalSubdomainsForThisProcessor);
void create_subdomain_and_write(const std::string &filename, unsigned subdomain,
int global_num_nodes, int global_num_elems,
int numSteps = -1, double timeStep = 0.0);

stk::mesh::MetaData& get_meta();
stk::mesh::BulkData& get_bulk();
SubdomainCreator& get_subdomain_creator();

int get_num_target_subdomains();

private:
void move_entities_into_mapped_subdomain_parts(const std::vector<unsigned>& mappings);
void change_parts_on_entities_on_all_subdomains(const std::vector<unsigned>& subdomain_proc_mapping);
std::vector<stk::mesh::Entity> get_entities_for_subdomain(size_t subdomain_num);
stk::mesh::EntityVector get_entitites_for_subdomain_using_field_from_buckets(size_t subdomain_num,
const stk::mesh::BucketVector& buckets);
Expand Down
62 changes: 36 additions & 26 deletions packages/stk/stk_balance/stk_balance/internal/SubdomainCreator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,13 @@
#include <stk_mesh/base/BulkData.hpp>
#include <stk_mesh/base/MetaData.hpp>
#include <stk_tools/mesh_clone/MeshClone.hpp>
#include <stk_tools/transfer_utils/TransientFieldTransferById.hpp>

namespace stk {
namespace balance {
namespace internal {

constexpr unsigned SubdomainCreator::INVALID_SUBDOMAIN;

SubdomainCreator::SubdomainCreator(stk::mesh::BulkData &bulk, int numTarget)
: mMeta(bulk.mesh_meta_data()),
mBulk(bulk),
Expand Down Expand Up @@ -69,13 +70,14 @@ std::string SubdomainCreator::getSubdomainPartName(int subdomainId)
return out.str();
}

void SubdomainCreator::declare_all_final_subdomain_parts()
stk::mesh::PartVector SubdomainCreator::declare_all_final_subdomain_parts()
{
for(int i=0;i<mNumFinalSubdomains;++i)
{
std::string partNameForSubdomain = getSubdomainPartName(i);
mMeta.declare_part(partNameForSubdomain, stk::topology::ELEMENT_RANK);
}
stk::mesh::PartVector subdomainParts;
for (int i = 0; i < mNumFinalSubdomains; ++i) {
std::string partNameForSubdomain = getSubdomainPartName(i);
subdomainParts.push_back(&mMeta.declare_part(partNameForSubdomain, stk::topology::ELEMENT_RANK));
}
return subdomainParts;
}

stk::mesh::Part* SubdomainCreator::get_subdomain_part(size_t subdomain_num)
Expand All @@ -99,17 +101,19 @@ stk::mesh::PartVector SubdomainCreator::get_parts_to_add_for_subdomain(size_t su

stk::io::EntitySharingInfo SubdomainCreator::get_node_sharing_info(unsigned subdomain)
{
stk::io::EntitySharingInfo nodeSharingInfo;
if (stk::transfer_utils::is_valid_subdomain(subdomain)) {
stk::mesh::EntityVector sharedNodes;
std::vector<int> procsForSharedNodes;
fill_shared_node_info_for_this_subdomain(subdomain, sharedNodes, procsForSharedNodes);

stk::io::EntitySharingInfo nodeSharingInfo;
for(size_t nodeIndex = 0; nodeIndex < sharedNodes.size(); nodeIndex++)
{
stk::mesh::EntityId nodeId = mBulk.identifier(sharedNodes[nodeIndex]);
nodeSharingInfo.push_back({nodeId, procsForSharedNodes[nodeIndex]});
stk::mesh::EntityId nodeId = mBulk.identifier(sharedNodes[nodeIndex]);
nodeSharingInfo.push_back({nodeId, procsForSharedNodes[nodeIndex]});
}
return nodeSharingInfo;
}
return nodeSharingInfo;
}

stk::mesh::EntityVector SubdomainCreator::get_nodes_shared_between_subdomains(int this_subdomain_index, int other_subdomain_index)
Expand Down Expand Up @@ -138,23 +142,29 @@ void SubdomainCreator::fill_shared_node_info_for_this_subdomain(const unsigned t
}
}

void SubdomainCreator::create_subdomain_and_write(const std::string &filename,
unsigned subdomain,
int global_num_nodes,
int global_num_elems,
const stk::io::EntitySharingInfo &nodeSharingInfo,
int numSteps, double timeStep)
void SubdomainCreator::create_subdomain_and_write(const std::string &filename, unsigned subdomain,
int global_num_nodes, int global_num_elems,
int numSteps, double timeStep)
{
stk::mesh::MetaData newMeta;
stk::mesh::BulkData newBulkData(newMeta, MPI_COMM_SELF);
stk::tools::copy_mesh(mBulk, *mMeta.get_part(getSubdomainPartName(subdomain)), newBulkData);

if(mTransientIo == nullptr) {
stk::io::write_file_for_subdomain(filename, subdomain, mNumFinalSubdomains, global_num_nodes, global_num_elems, newBulkData, nodeSharingInfo, numSteps, timeStep);
} else {
mTransientIo->setup_subdomain(newBulkData, filename, subdomain, nodeSharingInfo, global_num_nodes, global_num_elems);
mTransientIo->transfer_and_write_transient_data(subdomain);
stk::mesh::MetaData newMeta;
stk::mesh::BulkData newBulkData(newMeta, MPI_COMM_SELF);

const stk::mesh::Selector subdomainSelector = stk::transfer_utils::is_valid_subdomain(subdomain) ? *mMeta.get_part(getSubdomainPartName(subdomain))
: stk::mesh::Selector();
stk::tools::copy_mesh(mBulk, subdomainSelector, newBulkData);

stk::io::EntitySharingInfo nodeSharingInfo = get_node_sharing_info(subdomain);

if (mTransientIo == nullptr) {
if (stk::transfer_utils::is_valid_subdomain(subdomain)) {
stk::io::write_file_for_subdomain(filename, subdomain, mNumFinalSubdomains, global_num_nodes, global_num_elems,
newBulkData, nodeSharingInfo, numSteps, timeStep);
}
}
else {
mTransientIo->setup_subdomain(newBulkData, filename, subdomain, nodeSharingInfo, global_num_nodes, global_num_elems);
mTransientIo->transfer_and_write_transient_data(subdomain);
}
}

}}}
10 changes: 6 additions & 4 deletions packages/stk/stk_balance/stk_balance/internal/SubdomainCreator.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@
#include <vector>
#include <stk_mesh/base/Types.hpp>
#include <stk_io/IossBridge.hpp>
#include <stk_tools/transfer_utils/MtoNTransientFieldTransferById.hpp>

namespace stk { namespace mesh { class MetaData; }}
namespace stk { namespace mesh { class BulkData; }}
namespace stk { namespace io { class StkMeshIoBroker; }}
namespace stk { namespace transfer_utils { class MtoNTransientFieldTransferById; }}

namespace stk {
namespace balance {
Expand All @@ -56,14 +56,16 @@ class SubdomainCreator
~SubdomainCreator();

int get_num_final_subdomains() const { return mNumFinalSubdomains; }
void declare_all_final_subdomain_parts();
stk::mesh::PartVector declare_all_final_subdomain_parts();
void move_entities_into_final_subdomain_part(size_t i, const stk::mesh::EntityVector &entities);

stk::io::EntitySharingInfo get_node_sharing_info(unsigned subdomain);
void create_subdomain_and_write(const std::string &filename, unsigned subdomain, int global_num_nodes,
int global_num_elems, const stk::io::EntitySharingInfo &nodeSharingInfo,
void create_subdomain_and_write(const std::string &filename, unsigned subdomain,
int global_num_nodes, int global_num_elems,
int numSteps = -1, double timeStep = 0.0);

static constexpr unsigned INVALID_SUBDOMAIN = stk::transfer_utils::INVALID_SUBDOMAIN;

private:
std::string getSubdomainPartName(int subdomainId);
stk::mesh::Part* get_subdomain_part(size_t subdomain_num);
Expand Down
35 changes: 17 additions & 18 deletions packages/stk/stk_balance/stk_balance/internal/SubdomainFileWriter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,26 +44,25 @@ int get_subdomain_index(int includeMe, stk::ParallelMachine comm)

void write_subdomain_files(stk::mesh::BulkData &bulk, int numTarget, int mySubdomain, const std::string& outputMesh)
{
stk::balance::internal::SubdomainCreator subdomainCreator(bulk, numTarget);
subdomainCreator.declare_all_final_subdomain_parts();
stk::mesh::EntityVector elements;
stk::mesh::get_entities(bulk, stk::topology::ELEM_RANK,
bulk.mesh_meta_data().locally_owned_part(), elements);
bulk.modification_begin();
if(mySubdomain >= 0)
subdomainCreator.move_entities_into_final_subdomain_part(mySubdomain, elements);
bulk.modification_end();
stk::balance::internal::SubdomainCreator subdomainCreator(bulk, numTarget);
subdomainCreator.declare_all_final_subdomain_parts();
stk::mesh::EntityVector elements;
stk::mesh::get_entities(bulk, stk::topology::ELEM_RANK,
bulk.mesh_meta_data().locally_owned_part(), elements);
bulk.modification_begin();
if (mySubdomain >= 0) {
subdomainCreator.move_entities_into_final_subdomain_part(mySubdomain, elements);
}
bulk.modification_end();

std::vector<size_t> counts;
stk::mesh::comm_mesh_counts(bulk, counts);
int global_num_nodes = counts[stk::topology::NODE_RANK];
int global_num_elems = counts[stk::topology::ELEM_RANK];
std::vector<size_t> counts;
stk::mesh::comm_mesh_counts(bulk, counts);
int global_num_nodes = counts[stk::topology::NODE_RANK];
int global_num_elems = counts[stk::topology::ELEM_RANK];

if(mySubdomain >= 0)
{
stk::io::EntitySharingInfo sharingInfo = subdomainCreator.get_node_sharing_info(mySubdomain);
subdomainCreator.create_subdomain_and_write(outputMesh, mySubdomain, global_num_nodes, global_num_elems, sharingInfo);
}
if (mySubdomain >= 0) {
subdomainCreator.create_subdomain_and_write(outputMesh, mySubdomain, global_num_nodes, global_num_elems);
}
}

}}}
Expand Down
4 changes: 2 additions & 2 deletions packages/stk/stk_balance/stk_balance/internal/balanceCoincidentElements.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,12 @@ void set_all_coincidents_to_min_destination(const stk::mesh::ElemElemGraph& grap
stk::mesh::impl::LocalId minCoinLocalId,
DecompositionChangeList& changeList)
{
stk::mesh::Entity minCoinElem = graph.get_entity_from_local_id(minCoinLocalId);
stk::mesh::Entity minCoinElem = graph.get_entity(minCoinLocalId);
int minCoinDestination = changeList.get_entity_destination(minCoinElem);
modify_list_to_keep_on_same_proc(elem, minCoinElem, minCoinDestination, changeList);
for(stk::mesh::impl::LocalId coin : coincidents)
{
stk::mesh::Entity coinElem = graph.get_entity_from_local_id(coin);
stk::mesh::Entity coinElem = graph.get_entity(coin);
modify_list_to_keep_on_same_proc(coinElem, minCoinElem, minCoinDestination, changeList);
}
}
Expand Down
Loading