diff --git a/app/form/SeasQDDiscreteGreenOperator.cpp b/app/form/SeasQDDiscreteGreenOperator.cpp index dd7f7b28..9819ded4 100644 --- a/app/form/SeasQDDiscreteGreenOperator.cpp +++ b/app/form/SeasQDDiscreteGreenOperator.cpp @@ -11,6 +11,23 @@ namespace fs = std::filesystem; namespace tndm { +GreensFunctionIndices::GreensFunctionIndices(SeasQDDiscreteGreenOperator const& op) : n_bs(1) { + slip_block_size = op.base::friction().slip_block_size(); + num_local_elements = op.base::adapter().num_local_elements(); + m_bs = op.base::adapter().traction_block_size(); + m = num_local_elements * m_bs; + n = num_local_elements * slip_block_size * n_bs; + comm = op.base::comm(); + MPI_Comm_rank(comm, &rank); + + mb_offset = 0; + nb_offset = 0; + MPI_Scan(&num_local_elements, &mb_offset, 1, MPIU_INT, MPI_SUM, comm); + mb_offset -= num_local_elements; + MPI_Scan(&n, &nb_offset, 1, MPIU_INT, MPI_SUM, comm); + nb_offset -= n; +} + SeasQDDiscreteGreenOperator::SeasQDDiscreteGreenOperator( std::unique_ptr dgop, std::unique_ptr adapter, std::unique_ptr friction, @@ -56,11 +73,7 @@ SeasQDDiscreteGreenOperator::SeasQDDiscreteGreenOperator( prepend_checkpoint_path(sprefix); checkpoint_enabled_ = true; } - if (!checkpoint_enabled_) { - compute_discrete_greens_function(); - } else { - get_discrete_greens_function(mesh); - } + get_discrete_greens_function(mesh); } SeasQDDiscreteGreenOperator::~SeasQDDiscreteGreenOperator() { @@ -112,93 +125,6 @@ void SeasQDDiscreteGreenOperator::update_traction(double time, BlockVector const CHKERRTHROW(VecAXPY(base::traction_.vec(), time, t_boundary_->vec())); } -void SeasQDDiscreteGreenOperator::compute_discrete_greens_function() { - auto slip_block_size = base::friction().slip_block_size(); - - PetscInt num_local_elements = base::adapter().num_local_elements(); - PetscInt m_bs = base::adapter().traction_block_size(); - PetscInt n_bs = 1; - PetscInt m = num_local_elements * m_bs; - PetscInt n = num_local_elements * slip_block_size * n_bs; - - MPI_Comm comm = base::comm(); - - int rank; - MPI_Comm_rank(comm, &rank); - - PetscInt mb_offset = 0; - PetscInt nb_offset = 0; - MPI_Scan(&num_local_elements, &mb_offset, 1, MPIU_INT, MPI_SUM, comm); - mb_offset -= num_local_elements; - MPI_Scan(&n, &nb_offset, 1, MPIU_INT, MPI_SUM, comm); - nb_offset -= n; - - CHKERRTHROW(MatCreateDense(comm, m, n, PETSC_DECIDE, PETSC_DECIDE, nullptr, &G_)); - CHKERRTHROW(MatSetBlockSizes(G_, m_bs, n_bs)); - - S_ = std::make_unique(slip_block_size, num_local_elements, comm); - t_boundary_ = std::make_unique(m_bs, num_local_elements, comm); - - auto scatter = Scatter(base::adapter().fault_map().scatter_plan()); - auto ghost = scatter.template recv_prototype(slip_block_size, ALIGNMENT); - - PetscInt N; - CHKERRTHROW(VecGetSize(S_->vec(), &N)); - - Stopwatch sw; - double solve_time = 0.0; - for (PetscInt i = 0; i < N; ++i) { - - if (rank == 0) { - std::cout << "Computing Green's function " << (i + 1) << "/" << N; - } - sw.start(); - CHKERRTHROW(VecZeroEntries(S_->vec())); - if (i >= nb_offset && i < nb_offset + m) { - PetscScalar one = 1.0; - CHKERRTHROW(VecSetValue(S_->vec(), i, one, INSERT_VALUES)); - } - S_->begin_assembly(); - S_->end_assembly(); - - scatter.begin_scatter(*S_, ghost); - scatter.wait_scatter(); - - auto S_view = LocalGhostCompositeView(*S_, ghost); - base::solve(0.0, S_view); - base::update_traction(S_view); - - auto traction_handle = base::traction_.begin_access_readonly(); - for (std::size_t faultNo = 0; faultNo < num_local_elements; ++faultNo) { - PetscInt g_m = mb_offset + faultNo; - PetscInt g_n = i; - auto traction_block = traction_handle.subtensor(slice{}, faultNo); - CHKERRTHROW( - MatSetValuesBlocked(G_, 1, &g_m, 1, &g_n, traction_block.data(), INSERT_VALUES)); - } - base::traction_.end_access_readonly(traction_handle); - solve_time += sw.stop(); - if (rank == 0) { - constexpr double Days = 3600.0 * 24.0; - constexpr double Hours = 3600.0; - constexpr double Minutes = 60.0; - double avg_time = solve_time / (i + 1); - double etl = avg_time * (N - i - 1); - double etl_d = std::floor(etl / Days); - etl -= etl_d * Days; - double etl_h = std::floor(etl / Hours); - etl -= etl_h * Hours; - double etl_m = std::floor(etl / Minutes); - etl -= etl_m * Minutes; - std::cout << " (" << etl_d << "d " << etl_h << "h " << etl_m << "m " << std::floor(etl) - << "s left)" << std::endl; - } - } - - CHKERRTHROW(MatAssemblyBegin(G_, MAT_FINAL_ASSEMBLY)); - CHKERRTHROW(MatAssemblyEnd(G_, MAT_FINAL_ASSEMBLY)); -} - void SeasQDDiscreteGreenOperator::compute_boundary_traction() { MPI_Comm comm = base::comm(); int rank; @@ -223,37 +149,53 @@ void SeasQDDiscreteGreenOperator::compute_boundary_traction() { } PetscInt SeasQDDiscreteGreenOperator::create_discrete_greens_function() { - auto slip_block_size = base::friction().slip_block_size(); - PetscInt M, N, n_gf; - PetscInt num_local_elements = base::adapter().num_local_elements(); - PetscInt m_bs = base::adapter().traction_block_size(); - PetscInt n_bs = 1; - PetscInt m = num_local_elements * m_bs; - PetscInt n = num_local_elements * slip_block_size * n_bs; - PetscInt mb_offset = 0; - PetscInt nb_offset = 0; + GreensFunctionIndices ind(*this); int rank; + PetscInt M, N, n_gf; MPI_Comm comm = base::comm(); CHKERRTHROW(PetscPrintf(comm, "create_discrete_greens_function()\n")); MPI_Comm_rank(comm, &rank); - MPI_Scan(&num_local_elements, &mb_offset, 1, MPIU_INT, MPI_SUM, comm); - mb_offset -= num_local_elements; - MPI_Scan(&n, &nb_offset, 1, MPIU_INT, MPI_SUM, comm); - nb_offset -= n; - CHKERRTHROW(MatCreateDense(comm, m, n, PETSC_DECIDE, PETSC_DECIDE, nullptr, &G_)); - CHKERRTHROW(MatSetBlockSizes(G_, m_bs, n_bs)); + CHKERRTHROW(MatCreateDense(comm, ind.m, ind.n, PETSC_DECIDE, PETSC_DECIDE, nullptr, &G_)); + CHKERRTHROW(MatSetBlockSizes(G_, ind.m_bs, ind.n_bs)); CHKERRTHROW(MatGetSize(G_, &M, &N)); CHKERRTHROW(PetscPrintf(comm, "Green's function operator size: %D x %D\n", M, N)); - S_ = std::make_unique(slip_block_size, num_local_elements, comm); - t_boundary_ = std::make_unique(m_bs, num_local_elements, comm); + S_ = std::make_unique(ind.slip_block_size, ind.num_local_elements, comm); + t_boundary_ = std::make_unique(ind.m_bs, ind.num_local_elements, comm); CHKERRTHROW(VecGetSize(S_->vec(), &n_gf)); return n_gf; } +void SeasQDDiscreteGreenOperator::back_up_file(std::string file_to_backup) { + int rank; + MPI_Comm_rank(base::comm(), &rank); + std::string new_filename = file_to_backup + ".tmp"; + std::string bu_filename = file_to_backup + ".bu"; + if (rank == 0) { + if (fs::exists(file_to_backup)) { + try { + if (fs::exists(bu_filename)) { + fs::remove(bu_filename); + } + fs::rename(file_to_backup, bu_filename); + } catch (fs::filesystem_error& e) { + std::cerr << "Error moving file: " << e.what() << std::endl; + } + } + } + MPI_Barrier(base::comm()); + if (rank == 0) { + try { + fs::rename(new_filename, file_to_backup); + } catch (fs::filesystem_error& e) { + std::cerr << "Error moving file: " << e.what() << std::endl; + } + } +} + void SeasQDDiscreteGreenOperator::write_discrete_greens_operator( LocalSimplexMesh const& mesh, PetscInt current_gf, PetscInt n_gf) { PetscViewer v; @@ -263,9 +205,10 @@ void SeasQDDiscreteGreenOperator::write_discrete_greens_operator( MPI_Comm_size(base::comm(), &commsize); CHKERRTHROW(PetscTime(&t0)); - CHKERRTHROW(PetscViewerBinaryOpen(PetscObjectComm((PetscObject)G_), - gf_operator_filename_.c_str(), FILE_MODE_WRITE, &v)); - + std::string new_filename = gf_operator_filename_ + ".tmp"; + CHKERRTHROW(PetscViewerBinaryOpen(PetscObjectComm((PetscObject)G_), new_filename.c_str(), + FILE_MODE_WRITE, &v)); + CHKERRTHROW(PetscViewerBinarySetUseMPIIO(v, PETSC_TRUE)); { PetscInt commsize_checkpoint = (PetscInt)commsize; CHKERRTHROW(PetscViewerBinaryWrite(v, &commsize_checkpoint, 1, PETSC_INT)); @@ -276,7 +219,11 @@ void SeasQDDiscreteGreenOperator::write_discrete_greens_operator( CHKERRTHROW(MatView(G_, v)); CHKERRTHROW(PetscViewerDestroy(&v)); + + back_up_file(gf_operator_filename_); + CHKERRTHROW(PetscTime(&t1)); + CHKERRTHROW(PetscPrintf(PetscObjectComm((PetscObject)G_), "write_discrete_greens_operator():matrix %1.2e (sec)\n", (double)(t1 - t0))); @@ -313,10 +260,13 @@ void SeasQDDiscreteGreenOperator ::write_facet_labels_IS( (const PetscInt*)idx_, PETSC_USE_POINTER, &is)); CHKERRTHROW(PetscTime(&t0)); - CHKERRTHROW(PetscViewerBinaryOpen(PetscObjectComm((PetscObject)G_), gf_facet_filename_.c_str(), + std::string new_filename = gf_facet_filename_ + ".tmp"; + CHKERRTHROW(PetscViewerBinaryOpen(PetscObjectComm((PetscObject)G_), new_filename.c_str(), FILE_MODE_WRITE, &v)); + CHKERRTHROW(PetscViewerBinarySetUseMPIIO(v, PETSC_TRUE)); CHKERRTHROW(ISView(is, v)); CHKERRTHROW(PetscViewerDestroy(&v)); + back_up_file(gf_facet_filename_); CHKERRTHROW(PetscTime(&t1)); CHKERRTHROW(PetscPrintf(PetscObjectComm((PetscObject)G_), "write_discrete_greens_operator():facets %1.2e (sec)\n", @@ -338,6 +288,7 @@ IS SeasQDDiscreteGreenOperator ::load_facet_labels_seq_IS(void) { CHKERRTHROW(PetscTime(&t0)); CHKERRTHROW( PetscViewerBinaryOpen(PETSC_COMM_SELF, gf_facet_filename_.c_str(), FILE_MODE_READ, &v)); + CHKERRTHROW(PetscViewerBinarySetUseMPIIO(v, PETSC_TRUE)); CHKERRTHROW(ISLoad(is, v)); CHKERRTHROW(PetscViewerDestroy(&v)); CHKERRTHROW(PetscTime(&t1)); @@ -492,6 +443,7 @@ PetscInt SeasQDDiscreteGreenOperator::load_discrete_greens_operator( CHKERRTHROW(PetscTime(&t0)); CHKERRTHROW(PetscViewerBinaryOpen(PetscObjectComm((PetscObject)G_), gf_operator_filename_.c_str(), FILE_MODE_READ, &v)); + CHKERRTHROW(PetscViewerBinarySetUseMPIIO(v, PETSC_TRUE)); CHKERRTHROW(PetscViewerBinaryRead(v, &commsize_checkpoint, 1, NULL, PETSC_INT)); MPI_Comm_size(comm, &commsize); @@ -536,19 +488,11 @@ PetscInt SeasQDDiscreteGreenOperator::load_discrete_greens_operator( void SeasQDDiscreteGreenOperator::partial_assemble_discrete_greens_function( LocalSimplexMesh const& mesh, PetscInt current_gf, PetscInt n_gf) { - auto slip_block_size = base::friction().slip_block_size(); + GreensFunctionIndices ind(*this); - PetscInt num_local_elements = base::adapter().num_local_elements(); - PetscInt m_bs = base::adapter().traction_block_size(); - PetscInt n_bs = 1; - PetscInt m = num_local_elements * m_bs; - PetscInt n = num_local_elements * slip_block_size * n_bs; - PetscInt mb_offset = 0; - PetscInt nb_offset = 0; PetscInt start = current_gf; PetscInt N = n_gf; Stopwatch sw; - double solve_time; int rank; MPI_Comm comm = base::comm(); @@ -556,24 +500,25 @@ void SeasQDDiscreteGreenOperator::partial_assemble_discrete_greens_function( return; MPI_Comm_rank(comm, &rank); - MPI_Scan(&num_local_elements, &mb_offset, 1, MPIU_INT, MPI_SUM, comm); - mb_offset -= num_local_elements; - MPI_Scan(&n, &nb_offset, 1, MPIU_INT, MPI_SUM, comm); - nb_offset -= n; auto scatter = Scatter(base::adapter().fault_map().scatter_plan()); - auto ghost = scatter.template recv_prototype(slip_block_size, ALIGNMENT); + auto ghost = scatter.template recv_prototype(ind.slip_block_size, ALIGNMENT); - CHKERRTHROW(PetscPrintf(PetscObjectComm((PetscObject)G_), - "partial_assemble_discrete_greens_function() [%D , %D)\n", start, N)); - solve_time = 0.0; + if (start > 0) { + CHKERRTHROW(PetscPrintf(PetscObjectComm((PetscObject)G_), + "partial_assemble_discrete_greens_function() [%D , %D)\n", start, + N)); + } + double solve_time = 0.0; + double solve_time_from_start = 0.0; for (PetscInt i = start; i < N; ++i) { - CHKERRTHROW(PetscPrintf(PetscObjectComm((PetscObject)G_), - "Computing Green's function %D / %D\n", i, N)); + if (rank == 0) { + std::cout << "Computing Green's function " << (i + 1) << "/" << N; + } sw.start(); CHKERRTHROW(VecZeroEntries(S_->vec())); - if (i >= nb_offset && i < nb_offset + m) { + if (i >= ind.nb_offset && i < ind.nb_offset + ind.m) { PetscScalar one = 1.0; CHKERRTHROW(VecSetValue(S_->vec(), i, one, INSERT_VALUES)); } @@ -588,25 +533,45 @@ void SeasQDDiscreteGreenOperator::partial_assemble_discrete_greens_function( base::update_traction(S_view); auto traction_handle = base::traction_.begin_access_readonly(); - for (std::size_t faultNo = 0; faultNo < num_local_elements; ++faultNo) { - PetscInt g_m = mb_offset + faultNo; + for (std::size_t faultNo = 0; faultNo < ind.num_local_elements; ++faultNo) { + PetscInt g_m = ind.mb_offset + faultNo; PetscInt g_n = i; auto traction_block = traction_handle.subtensor(slice{}, faultNo); CHKERRTHROW( MatSetValuesBlocked(G_, 1, &g_m, 1, &g_n, traction_block.data(), INSERT_VALUES)); } base::traction_.end_access_readonly(traction_handle); - solve_time += sw.stop(); + double step_time = sw.stop(); + solve_time += step_time; + solve_time_from_start += step_time; + + if (rank == 0) { + constexpr double Days = 3600.0 * 24.0; + constexpr double Hours = 3600.0; + constexpr double Minutes = 60.0; + double avg_time = solve_time_from_start / (i + 1 - start); + double etl = avg_time * (N - i - 1); + double etl_d = std::floor(etl / Days); + etl -= etl_d * Days; + double etl_h = std::floor(etl / Hours); + etl -= etl_h * Hours; + double etl_m = std::floor(etl / Minutes); + etl -= etl_m * Minutes; + std::cout << " (" << etl_d << "d " << etl_h << "h " << etl_m << "m " << std::floor(etl) + << "s left)" << std::endl; + } current_gf = i + 1; - /* checkpoint */ - MPI_Bcast(&solve_time, 1, MPI_DOUBLE, 0, comm); - if (solve_time / 60.0 > checkpoint_every_nmins_) { - CHKERRTHROW(MatAssemblyBegin(G_, MAT_FINAL_ASSEMBLY)); - CHKERRTHROW(MatAssemblyEnd(G_, MAT_FINAL_ASSEMBLY)); - write_discrete_greens_operator(mesh, current_gf, n_gf); - solve_time = 0.0; + if (checkpoint_enabled_) { + /* checkpoint */ + MPI_Bcast(&solve_time, 1, MPI_DOUBLE, 0, comm); + if (solve_time / 60.0 > checkpoint_every_nmins_) { + CHKERRTHROW(MatAssemblyBegin(G_, MAT_FINAL_ASSEMBLY)); + CHKERRTHROW(MatAssemblyEnd(G_, MAT_FINAL_ASSEMBLY)); + write_discrete_greens_operator(mesh, current_gf, n_gf); + solve_time = 0.0; + } } } @@ -624,18 +589,22 @@ void SeasQDDiscreteGreenOperator::get_discrete_greens_function( n_gf = create_discrete_greens_function(); } - // If a checkpoint file is found, load it. Record the number of assembled GFs found in file. - CHKERRTHROW(PetscTestFile(gf_operator_filename_.c_str(), 'r', &found)); - if (found) { - n_gfloaded = load_discrete_greens_operator(mesh, n_gf); + if (checkpoint_enabled_) { + // If a checkpoint file is found, load it. Record the number of assembled GFs found in file. + CHKERRTHROW(PetscTestFile(gf_operator_filename_.c_str(), 'r', &found)); + if (found) { + n_gfloaded = load_discrete_greens_operator(mesh, n_gf); + } } // Assemble as many GFs as possible in the range [current_gf, n_gf) partial_assemble_discrete_greens_function(mesh, n_gfloaded, n_gf); - // Write out the operator whenever the fully assembled operator was not loaded from file - if (n_gfloaded != n_gf) { - write_discrete_greens_operator(mesh, n_gf, n_gf); + if (checkpoint_enabled_) { + // Write out the operator whenever the fully assembled operator was not loaded from file + if (n_gfloaded != n_gf) { + write_discrete_greens_operator(mesh, n_gf, n_gf); + } } } diff --git a/app/form/SeasQDDiscreteGreenOperator.h b/app/form/SeasQDDiscreteGreenOperator.h index d632a08b..ee013151 100644 --- a/app/form/SeasQDDiscreteGreenOperator.h +++ b/app/form/SeasQDDiscreteGreenOperator.h @@ -18,6 +18,23 @@ namespace tndm { +class SeasQDDiscreteGreenOperator; + +struct GreensFunctionIndices { + PetscInt slip_block_size; + PetscInt num_local_elements; + PetscInt m_bs; // traction block size + PetscInt n_bs; // always 1 + PetscInt m; + PetscInt n; + PetscInt mb_offset; + PetscInt nb_offset; + int rank; + MPI_Comm comm; + + GreensFunctionIndices(SeasQDDiscreteGreenOperator const& op); +}; + class SeasQDDiscreteGreenOperator : public SeasQDOperator { public: using base = SeasQDOperator; @@ -59,7 +76,6 @@ class SeasQDDiscreteGreenOperator : public SeasQDOperator { void update_traction(double time, BlockVector const& state); private: - void compute_discrete_greens_function(); void compute_boundary_traction(); PetscInt create_discrete_greens_function(); void partial_assemble_discrete_greens_function(LocalSimplexMesh const& mesh, @@ -70,6 +86,7 @@ class SeasQDDiscreteGreenOperator : public SeasQDOperator { PetscInt n_gf_); // all logic associated with matix craetion, loading / partial assembly is done here void get_discrete_greens_function(LocalSimplexMesh const& mesh); + void back_up_file(std::string file_to_backup); void write_discrete_greens_traction(); void load_discrete_greens_traction(); void get_boundary_traction(); diff --git a/app/tandem/SEAS.cpp b/app/tandem/SEAS.cpp index 53554f75..1e11169a 100644 --- a/app/tandem/SEAS.cpp +++ b/app/tandem/SEAS.cpp @@ -146,10 +146,21 @@ struct operator_specifics static auto make(LocalSimplexMesh const& mesh, Config const& cfg, seas::ContextBase& ctx) { + auto const& cfgcp = cfg.gf_checkpoint_config; + + std::optional prefix; + double freq_cputime; + if (!cfgcp) { + prefix = std::nullopt; + freq_cputime = 1e10; + } else { + prefix = cfgcp->prefix; + freq_cputime = cfgcp->frequency_cputime_minutes; + } + auto seasop = std::make_shared( - std::move(ctx.dg()), std::move(ctx.adapter()), std::move(ctx.friction()), mesh, - cfg.gf_checkpoint_prefix, cfg.gf_checkpoint_every_nmins, cfg.matrix_free, - MGConfig(cfg.mg_coarse_level, cfg.mg_strategy)); + std::move(ctx.dg()), std::move(ctx.adapter()), std::move(ctx.friction()), mesh, prefix, + freq_cputime, cfg.matrix_free, MGConfig(cfg.mg_coarse_level, cfg.mg_strategy)); ctx.setup_seasop(*seasop); seasop->warmup(); return seasop; diff --git a/app/tandem/SeasConfig.cpp b/app/tandem/SeasConfig.cpp index 4778d5ea..0f7cef91 100644 --- a/app/tandem/SeasConfig.cpp +++ b/app/tandem/SeasConfig.cpp @@ -31,6 +31,17 @@ template void setOutputConfigSchema(TableSchema& out .help("Maximum time difference between samples."); } +template +void setGfCheckpointConfigSchema(TableSchema& gfCheckpointSchema) { + gfCheckpointSchema.add_value("prefix", up_cast(&Derived::prefix)) + .default_value("gf_checkpoint") + .help("Path where Green's function operator and RHS will be checkpointed."); + gfCheckpointSchema + .add_value("freq_cputime", up_cast(&Derived::frequency_cputime_minutes)) + .default_value(30.0) + .help("CPU time (minutes) frequency between Green's function operator checkpoints"); +} + template void setDomainOutputConfigSchema(TableSchema& outputSchema) { setOutputConfigSchema(outputSchema); @@ -124,13 +135,6 @@ void setConfigSchema(TableSchema& schema, .default_value(false) .help("Assert that boundary is a linear function of time (i.e. boundary(x, t) = f(x) t)."); - schema.add_value("gf_checkpoint_prefix", &Config::gf_checkpoint_prefix) - .help("Path where Green's function operator and RHS will be checkpointed."); - schema.add_value("gf_checkpoint_every_nmins", &Config::gf_checkpoint_every_nmins) - .default_value(30.0) - .help("time interval, in minutes, at which the Green's function operator data is saved to " - "disk."); - schema.add_value("matrix_free", &Config::matrix_free) .default_value(false) .help("Use matrix-free operators."); @@ -169,6 +173,9 @@ void setConfigSchema(TableSchema& schema, auto& domainProbeOutputSchema = schema.add_table("domain_probe_output", &Config::domain_probe_output); detail::setProbeOutputConfigSchema(domainProbeOutputSchema); + + auto& gfCheckpointSchema = schema.add_table("gf_checkpoint", &Config::gf_checkpoint_config); + detail::setGfCheckpointConfigSchema(gfCheckpointSchema); } } // namespace tndm diff --git a/app/tandem/SeasConfig.h b/app/tandem/SeasConfig.h index 31b27a76..d32606f1 100644 --- a/app/tandem/SeasConfig.h +++ b/app/tandem/SeasConfig.h @@ -57,6 +57,11 @@ struct ProbeOutputConfig : TabularOutputConfig { std::vector> probes; }; +struct GfCheckpointConfig { + std::string prefix; + double frequency_cputime_minutes; +}; + struct Config { std::optional resolution; double final_time; @@ -75,15 +80,13 @@ struct Config { MGStrategy mg_strategy; unsigned mg_coarse_level; - std::optional gf_checkpoint_prefix; - double gf_checkpoint_every_nmins; - std::optional> generate_mesh; std::optional fault_output; std::optional fault_scalar_output; std::optional domain_output; std::optional fault_probe_output; std::optional domain_probe_output; + std::optional gf_checkpoint_config; }; void setConfigSchema(TableSchema& schema, diff --git a/docs/first-model/parameters.rst b/docs/first-model/parameters.rst index 34c5c233..b330b851 100644 --- a/docs/first-model/parameters.rst +++ b/docs/first-model/parameters.rst @@ -17,11 +17,14 @@ Tandem simulation parameters are written in the toml script. Here are the key pa - **mg_coarse_level**: Polynomial degree of coarsest MG level. Default = 1. - **mg_strategy**: MG level selection strategy. Available options: [TwoLevel | Logarithmic | Full]. Default = TwoLevel. -When using :code:`mode=QDGreen`, you can use Green's function checkpointing feature by defining the following parameters: -- **gf_checkpoint_prefix**: Path where Green's function operator and RHS will be checkpointed. -- **gf_checkpoint_every_nmins**: Time interval, in minutes, at which the Green's function operator data is saved to disk. Default = 30.0. +Green's function operator checkpointing +--------------------------------------- +When using :code:`mode=QDGreen`, Green's function checkpointing is parameterized by the **[gf_checkpoint]** section. Here are the key parameters and their descriptions: + +- **prefix**: Path where Green's function operator and RHS will be checkpointed. +- **freq_cputime**: CPU time (minutes) frequency between Green's function operator checkpoints. Output configurations --------------------- @@ -86,4 +89,4 @@ Commented parameter file: # Domain outputs [domain_output] prefix = "output/domain" - rtol = 0.1 \ No newline at end of file + rtol = 0.1 diff --git a/examples/tandem/2d/BP6/bp6_A.toml b/examples/tandem/2d/BP6/bp6_A.toml index b29f913e..8438d37a 100644 --- a/examples/tandem/2d/BP6/bp6_A.toml +++ b/examples/tandem/2d/BP6/bp6_A.toml @@ -7,7 +7,8 @@ scenario = "bp6" ref_normal = [-1, 0] boundary_linear = true -gf_checkpoint_prefix = "GreensFunctions/bp6_hf250" +[gf_checkpoint] +prefix = "GreensFunctions/bp6_hf250" [fault_output] prefix = "outputs_A_250m/fault" @@ -420,4 +421,4 @@ probes = [ { name = "+198", x = [0.0, 19.8] }, { name = "+199", x = [0.0, 19.9] }, { name = "+200", x = [0.0, 20.0] } -] \ No newline at end of file +] diff --git a/examples/tandem/2d/BP6/bp6_S.toml b/examples/tandem/2d/BP6/bp6_S.toml index 47f091b9..9125ef1f 100644 --- a/examples/tandem/2d/BP6/bp6_S.toml +++ b/examples/tandem/2d/BP6/bp6_S.toml @@ -7,7 +7,8 @@ scenario = "bp6" ref_normal = [-1, 0] boundary_linear = true -gf_checkpoint_prefix = "GreensFunctions/bp6_hf50" +[gf_checkpoint] +prefix = "GreensFunctions/bp6_hf50" [fault_output] prefix = "outputs_S_50m/fault" @@ -420,4 +421,4 @@ probes = [ { name = "+198", x = [0.0, 19.8] }, { name = "+199", x = [0.0, 19.9] }, { name = "+200", x = [0.0, 20.0] } -] \ No newline at end of file +]