Skip to content

Commit

Permalink
CPU - include zlib and compressed dh files
Browse files Browse the repository at this point in the history
  • Loading branch information
@ZamanLantra
ZamanLantra committed Oct 30, 2024
1 parent 211981f commit 1469ce0
Show file tree
Hide file tree
Showing 13 changed files with 566 additions and 164 deletions.
7 changes: 3 additions & 4 deletions app_fempic_cg/fempic.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ int main(int argc, char **argv)
std::string log = "";
const OPP_BOOL print_final_log = opp_params->get<OPP_BOOL>("print_final");
int64_t total_part_iter = 0;
const OPP_REAL expansion[3] = { 4*grid_spacing, 4*grid_spacing, 4*grid_spacing };
const opp_point expansion(4*grid_spacing, 4*grid_spacing, 4*grid_spacing);

std::shared_ptr<DataPointers> m = load_mesh();

Expand Down Expand Up @@ -140,8 +140,7 @@ int main(int argc, char **argv)
opp_inc_part_count_with_distribution(particle_set, inject_count, if_distrib, false);

// these two lines are only required if we plan to use direct_hop
opp::BoundingBox bounding_box = opp::BoundingBox(n_pos, DIM, expansion);
opp_init_direct_hop(grid_spacing, DIM, c_gbl_id, bounding_box);
opp_init_direct_hop(grid_spacing, c_gbl_id, opp::BoundingBox(n_pos, DIM, expansion));

auto field_solver = std::make_unique<FESolver>(c2n_map, n_type, n_pos, n_bnd_pot);
field_solver->enrich_cell_shape_deriv(c_sd);
Expand Down Expand Up @@ -243,7 +242,7 @@ int main(int argc, char **argv)
}

//*************************************************************************************************
// std::string f = std::string("F_") + std::to_string(ts + 1);
// std::string f = std::string("F_") + std::to_string(OPP_main_loop_iter + 1);
// opp_print_map_to_txtfile(c2n_map , f.c_str(), "c2n_map.dat");
// opp_print_dat_to_txtfile(n_charge_den, f.c_str(), "n_charge_den.dat");
// opp_mpi_print_dat_to_txtfile(c_sd, "c_sd.dat");
10 changes: 5 additions & 5 deletions app_fempic_cg/fempic_opp.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

// Auto-generated at 2024-10-10 12:37:31.766026 by opp-translator
// Auto-generated at 2024-10-17 19:08:48.174421 by opp-translator
/*
BSD 3-Clause License
Expand Down Expand Up @@ -46,7 +46,7 @@ void opp_par_loop_all__compute_node_charge_density_kernel(opp_set,opp_iterate_ty
void opp_par_loop_all__compute_electric_field_kernel(opp_set,opp_iterate_type,opp_arg,opp_arg,opp_arg,opp_arg,opp_arg,opp_arg);
void opp_par_loop_all__get_max_cef_kernel(opp_set,opp_iterate_type,opp_arg,opp_arg);
void opp_par_loop_all__get_final_max_values_kernel(opp_set,opp_iterate_type,opp_arg,opp_arg,opp_arg,opp_arg);
void opp_init_direct_hop_cg(double,int,const opp_dat,const opp::BoundingBox&,opp_map,opp_map,opp_arg,opp_arg,opp_arg,opp_arg);
void opp_init_direct_hop_cg(double,const opp_dat,const opp::BoundingBox&,opp_map,opp_map,opp_arg,opp_arg,opp_arg,opp_arg);

#include "fempic_misc_mesh_loader.h"
#include "fempic_misc.h"
Expand All @@ -73,7 +73,7 @@ int main(int argc, char **argv)
std::string log = "";
const OPP_BOOL print_final_log = opp_params->get<OPP_BOOL>("print_final");
int64_t total_part_iter = 0;
const OPP_REAL expansion[3] = { 4*grid_spacing, 4*grid_spacing, 4*grid_spacing };
const opp_point expansion(4*grid_spacing, 4*grid_spacing, 4*grid_spacing);

std::shared_ptr<DataPointers> m = load_mesh();

Expand Down Expand Up @@ -145,8 +145,8 @@ int main(int argc, char **argv)
opp_inc_part_count_with_distribution(particle_set, inject_count, if_distrib, false);

// these two lines are only required if we plan to use direct_hop
opp::BoundingBox bounding_box = opp::BoundingBox(n_pos, DIM, expansion);
opp_init_direct_hop_cg(grid_spacing, DIM, c_gbl_id, bounding_box, c2c_map, p2c_map,
opp_init_direct_hop_cg(grid_spacing, c_gbl_id, opp::BoundingBox(n_pos, DIM, expansion),
c2c_map, p2c_map,
opp_arg_dat(p_pos, OPP_READ),
opp_arg_dat(p_lc, OPP_WRITE),
opp_arg_dat(c_volume, p2c_map, OPP_READ),
Expand Down
40 changes: 37 additions & 3 deletions app_fempic_cg/mpi/move_kernel_loop.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
// AUTO GENERATED CODE
//*********************************************

#define X_HOPS 5

namespace opp_k4 {
inline void move_kernel(
const double *point_pos, double* point_lc,
Expand Down Expand Up @@ -79,6 +81,7 @@ void opp_particle_move__move_kernel(opp_set set, opp_map c2c_map, opp_map p2c_ma
OPP_mesh_relation_data = (OPP_INT*)p2c_map->p2c_dat->data;
#ifdef LOG_HOPS
OPP_move_max_hops = 0;
OPP_move_moreX_hops = 0;
#endif

opp_mpi_halo_exchanges(set, nargs, args);
Expand Down Expand Up @@ -116,6 +119,7 @@ void opp_particle_move__move_kernel(opp_set set, opp_map c2c_map, opp_map p2c_ma

#ifdef LOG_HOPS
OPP_move_max_hops = (OPP_move_max_hops < hops) ? hops : OPP_move_max_hops;
if (hops > X_HOPS) OPP_move_moreX_hops++;
#endif
};

Expand Down Expand Up @@ -214,7 +218,7 @@ void opp_particle_move__move_kernel(opp_set set, opp_map c2c_map, opp_map p2c_ma
opp_profiler->end("move_kernel");
}

void opp_init_direct_hop_cg(double grid_spacing, int dim, const opp_dat c_gbl_id, const opp::BoundingBox& b_box,
void opp_init_direct_hop_cg(double grid_spacing, const opp_dat c_gbl_id, const opp::BoundingBox& b_box,
opp_map c2c_map, opp_map p2c_map,
opp_arg arg0, // p_pos | OPP_READ
opp_arg arg1, // p_lc | OPP_WRITE
Expand Down Expand Up @@ -276,8 +280,38 @@ void opp_init_direct_hop_cg(double grid_spacing, int dim, const opp_dat c_gbl_id
}
}
};

cellMapper->generateStructuredMesh(c_gbl_id->set, c_gbl_id, all_cell_checker);

if (opp_params->get<OPP_BOOL>("opp_dh_data_generate")) {
cellMapper->generateStructuredMesh(c_gbl_id->set, c_gbl_id, all_cell_checker);

// auto cellMapper2 = std::make_shared<opp::CellMapper>(boundingBox, grid_spacing, comm);
// cellMapper2->generateStructuredMeshFromFile(c_gbl_id->set, c_gbl_id);
// cellMapper2->waitBarrier();
// if (OPP_rank == 0)
// {
// for (size_t i = 0; i < cellMapper->globalGridSize; i++) {
// if (cellMapper->structMeshToCellMapping[i] != cellMapper2->structMeshToCellMapping[i]) {
// opp_printf("APP", "Incorrect value from cell file at %d - system %d - file %d",
// i, cellMapper->structMeshToCellMapping[i], cellMapper2->structMeshToCellMapping[i]);
// }
// }
// int count = 0, non_root = 0;
// for (size_t i = 0; i < cellMapper->globalGridSize; i++) {
// if (cellMapper->structMeshToRankMapping[i] != cellMapper2->structMeshToRankMapping[i]) { count++;
// opp_printf("APP", "Incorrect value from rank file at %d - cid %d - system %d - file %d",
// i, cellMapper->structMeshToCellMapping[i], cellMapper->structMeshToRankMapping[i], cellMapper2->structMeshToRankMapping[i]);
// }

// if (cellMapper2->structMeshToRankMapping[i] != 0 && cellMapper2->structMeshToRankMapping[i] != MAX_CELL_INDEX) {
// non_root++;
// }
// }
// opp_printf("APP", "Incorrect rank values %d non root %d", count, non_root);
// }
}
else {
cellMapper->generateStructuredMeshFromFile(c_gbl_id->set, c_gbl_id);
}

opp_profiler->reg("GlbToLocal");
opp_profiler->reg("GblMv_Move");
Expand Down
33 changes: 30 additions & 3 deletions app_fempic_cg/omp/move_kernel_loop.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
// AUTO GENERATED CODE
//*********************************************

#define X_HOPS 5

namespace opp_k4 {
inline void move_kernel(
char& opp_move_status_flag, const bool opp_move_hop_iter_one_flag, // Added by code-gen
Expand Down Expand Up @@ -87,6 +89,12 @@ void opp_particle_move__move_kernel(opp_set set, opp_map c2c_map, opp_map p2c_ma

opp_mpi_halo_wait_all(nargs, args);

#ifdef LOG_HOPS
std::vector<int> int_hops(nthreads, 0);
std::vector<int> moreX_hops(nthreads, 0);
OPP_move_moreX_hops = 0;
#endif

// lambda function for multi hop particle movement
auto multihop_mover = [&](const int n, const int thread) {

Expand All @@ -100,6 +108,9 @@ void opp_particle_move__move_kernel(opp_set set, opp_map c2c_map, opp_map p2c_ma
char move_flag = OPP_MOVE_DONE;
bool iter_one_flag = true;

#ifdef LOG_HOPS
int hops = 0;
#endif
do {
move_flag = OPP_MOVE_DONE;
opp_c2c = c2c_map->map + (opp_p2c[0] * 4);
Expand All @@ -111,8 +122,15 @@ void opp_particle_move__move_kernel(opp_set set, opp_map c2c_map, opp_map p2c_ma
(const OPP_REAL *)args[2].data + (opp_p2c[0] * 1),
(const OPP_REAL *)args[3].data + (opp_p2c[0] * 16)
);

#ifdef LOG_HOPS
hops++;
#endif
} while (opp_check_part_move_status(move_flag, iter_one_flag, opp_p2c[0], n, thread));

#ifdef LOG_HOPS
int_hops[thread] = (int_hops[thread] < hops) ? hops : int_hops[thread];
if (hops > X_HOPS) moreX_hops[thread]++;
#endif
};

// ----------------------------------------------------------------------------
Expand Down Expand Up @@ -241,13 +259,17 @@ void opp_particle_move__move_kernel(opp_set set, opp_map c2c_map, opp_map p2c_ma
opp_profiler->end(profName);
}

#ifdef LOG_HOPS
OPP_move_max_hops = *std::max_element(int_hops.begin(), int_hops.end());
OPP_move_moreX_hops = std::accumulate(moreX_hops.begin(), moreX_hops.end(), 0);
#endif

opp_set_dirtybit(nargs, args);

opp_profiler->end("move_kernel");
}

void opp_init_direct_hop_cg(double grid_spacing, int dim, const opp_dat c_gbl_id, const opp::BoundingBox& b_box,
void opp_init_direct_hop_cg(double grid_spacing, const opp_dat c_gbl_id, const opp::BoundingBox& b_box,
opp_map c2c_map, opp_map p2c_map,
opp_arg arg0, // p_pos | OPP_READ
opp_arg arg1, // p_lc | OPP_WRITE
Expand Down Expand Up @@ -311,7 +333,12 @@ void opp_init_direct_hop_cg(double grid_spacing, int dim, const opp_dat c_gbl_id
}
};

cellMapper->generateStructuredMesh(c_gbl_id->set, c_gbl_id, all_cell_checker);
if (opp_params->get<OPP_BOOL>("opp_dh_data_generate")) {
cellMapper->generateStructuredMesh(c_gbl_id->set, c_gbl_id, all_cell_checker);
}
else {
cellMapper->generateStructuredMeshFromFile(c_gbl_id->set, c_gbl_id);
}

opp_profiler->reg("GlbToLocal");
opp_profiler->reg("GblMv_Move");
Expand Down
2 changes: 2 additions & 0 deletions opp_lib/include/opp_defs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,8 @@ extern size_t OPP_gpu_shared_mem_per_block;
extern int OPP_part_cells_set_size;
extern int OPP_part_comm_count_per_iter;
extern int OPP_move_max_hops;
extern int OPP_move_moreX_hops;
extern bool OPP_dh_data_dump;

extern std::vector<opp_set> opp_sets;
extern std::vector<opp_map> opp_maps;
Expand Down
49 changes: 29 additions & 20 deletions opp_lib/include/opp_direct_hop_core.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ class BoundingBox {
opp_point globalBoundingBox[2]; // index 0 is min, index 1 is max

public:
opp_point domain_expansion;

//***********************************
BoundingBox(int dim, opp_point minCoordinate, opp_point maxCoordinate) : dim(dim) {

Expand All @@ -65,7 +67,9 @@ class BoundingBox {
}

//***********************************
BoundingBox(const opp_dat node_pos_dat, int dim, const OPP_REAL expansion[3]) : dim(dim) {
BoundingBox(const opp_dat node_pos_dat, int dim, const opp_point expansion)
: dim(dim), domain_expansion(expansion) {

if (dim != 3 && dim != 2) {
opp_abort(std::string("For now, only 2D/3D BoundingBox is implemented"));
}
Expand Down Expand Up @@ -97,13 +101,13 @@ class BoundingBox {

// Why BOUND_OFFSET? to overcome overlapping == nodes
if (node_count != 0) {
minCoordinate.x -= (expansion[0] + BOUND_OFFSET);
minCoordinate.y -= (expansion[0] + BOUND_OFFSET);
maxCoordinate.x += (expansion[1] - BOUND_OFFSET);
maxCoordinate.y += (expansion[1] - BOUND_OFFSET);
minCoordinate.x -= (domain_expansion.x + BOUND_OFFSET);
minCoordinate.y -= (domain_expansion.x + BOUND_OFFSET);
maxCoordinate.x += (domain_expansion.y - BOUND_OFFSET);
maxCoordinate.y += (domain_expansion.y - BOUND_OFFSET);
if (dim == 3) {
minCoordinate.z -= (expansion[2] + BOUND_OFFSET);
maxCoordinate.z += (expansion[2] - BOUND_OFFSET);
minCoordinate.z -= (domain_expansion.z + BOUND_OFFSET);
maxCoordinate.z += (domain_expansion.z - BOUND_OFFSET);
}
}

Expand All @@ -124,6 +128,7 @@ class BoundingBox {
this->boundingBox[1] = other.boundingBox[1];
this->globalBoundingBox[0] = other.globalBoundingBox[0];
this->globalBoundingBox[1] = other.globalBoundingBox[1];
this->domain_expansion = other.domain_expansion;
}

//***********************************
Expand Down Expand Up @@ -211,43 +216,46 @@ class GlobalToLocalCellIndexMapper {

public:
//***********************************
GlobalToLocalCellIndexMapper(const opp_dat global_cell_id_dat) {
if (OPP_rank == 0)
opp_printf("GlobalToLocalCellIndexMapper", "START");
GlobalToLocalCellIndexMapper(const opp_dat global_cell_id_dat, bool inc_halo = true) {

globalToLocalCellIndexMap.clear();

const opp_set cells_set = global_cell_id_dat->set;
const OPP_INT size_inc_halo = (cells_set->size + cells_set->exec_size + cells_set->nonexec_size);
const opp_set c_set = global_cell_id_dat->set;
const OPP_INT size = inc_halo ?
(c_set->size + c_set->exec_size + c_set->nonexec_size) : (c_set->size);

for (OPP_INT i = 0; i < size_inc_halo; i++) {
if (OPP_rank == 0)
opp_printf("GlobalToLocalCellIndexMapper", "Map Size %d", size);

for (OPP_INT i = 0; i < size; i++) {
const OPP_INT glbIndex = ((OPP_INT*)global_cell_id_dat->data)[i];
globalToLocalCellIndexMap.insert(std::make_pair(glbIndex, i));
}

if (OPP_rank == 0)
if (OPP_DBG && OPP_rank == 0)
opp_printf("GlobalToLocalCellIndexMapper", "END");
}

//***********************************
~GlobalToLocalCellIndexMapper() {
globalToLocalCellIndexMap.clear();

if (OPP_rank == 0)
if (OPP_DBG && OPP_rank == 0)
opp_printf("~GlobalToLocalCellIndexMapper", "DONE");
}

//***********************************
OPP_INT map(const OPP_INT globalIndex) {
OPP_INT map(const OPP_INT globalIndex) const {
if (globalIndex == MAX_CELL_INDEX)
return MAX_CELL_INDEX;

auto it = globalToLocalCellIndexMap.find(globalIndex);
if (it != globalToLocalCellIndexMap.end())
return it->second;

opp_printf("GlobalToLocalCellIndexMapper",
"Error: Local cell index not found for global index %d", globalIndex);
if (OPP_DBG)
opp_printf("GlobalToLocalCellIndexMapper",
"Error: Local cell index not found for global index %d", globalIndex);
return MAX_INT;
}
};
Expand Down Expand Up @@ -297,10 +305,11 @@ class CellMapper {
void reduceInterNodeMappings(int callID);
void createStructMeshMappingArrays();
void convertToLocalMappings(const opp_dat global_cell_id_dat);
void convertToLocalMappingsIncRank(const opp_dat global_cell_id_dat);
void hostToDeviceTransferMappings();
void generateStructuredMesh(opp_set set, const opp_dat c_gbl_id,
const std::function<void(const opp_point&, int&)>& all_cell_checker);

void generateStructuredMeshFromFile(opp_set set, const opp_dat c_gbl_id);
//***********************************
inline void init_host(const double gridSpacing)
{
Expand Down Expand Up @@ -500,7 +509,7 @@ class CellMapper {
#endif
}

inline void convertToLocalMappings_seq(GlobalToLocalCellIndexMapper mapper)
inline void convertToLocalMappings_seq(const GlobalToLocalCellIndexMapper& mapper)
{
for (size_t i = 0; i < globalGridSize; i++) {
if (structMeshToRankMapping[i] == OPP_rank) {
Expand Down
6 changes: 5 additions & 1 deletion opp_lib/include/opp_util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,11 @@ int file_exist(char const *filename);

bool opp_type_equivalence(const char *a, const char *b);


void opp_compress_write(const std::string &filename,
const int* data, const size_t count);
void opp_decompress_read(const std::string &filename, size_t originalSize,
int* data);

//*************************************************************************************************
template <typename RNG>
inline std::vector<std::vector<double>>
Expand Down
Loading

0 comments on commit 1469ce0

Please sign in to comment.