From f6a4110d5e808786c246ff47d444258d3334b8e6 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 15 Aug 2023 22:17:25 -0400 Subject: [PATCH 01/13] added arg; redo init writers logic --- src/cmdstan/arguments/arg_pathfinder.hpp | 2 + src/cmdstan/command.hpp | 16 ++------ src/cmdstan/command_helper.hpp | 52 ++++++++++++++++++------ 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/cmdstan/arguments/arg_pathfinder.hpp b/src/cmdstan/arguments/arg_pathfinder.hpp index e5dda99ad0..09d1795ddc 100644 --- a/src/cmdstan/arguments/arg_pathfinder.hpp +++ b/src/cmdstan/arguments/arg_pathfinder.hpp @@ -16,6 +16,8 @@ class arg_pathfinder : public arg_lbfgs { _subarguments.push_back(new arg_single_int_pos( "num_psis_draws", "Number of draws from PSIS sample", 1000)); + _subarguments.push_back( + new arg_single_bool("save_single_paths", "Output single-path pathfinder draws as CSV", false)); _subarguments.push_back( new arg_single_int_pos("num_paths", "Number of single pathfinders", 4)); _subarguments.push_back(new arg_single_int_pos( diff --git a/src/cmdstan/command.hpp b/src/cmdstan/command.hpp index 8c2047275f..0b3f901866 100644 --- a/src/cmdstan/command.hpp +++ b/src/cmdstan/command.hpp @@ -195,8 +195,10 @@ int command(int argc, const char *argv[]) { diagnostic_csv_writers; std::vector> diagnostic_json_writers; + stan::callbacks::unique_stream_writer pathfinder_writer; + init_callbacks(parser, sample_writers, diagnostic_csv_writers, - diagnostic_json_writers); + diagnostic_json_writers, pathfinder_writer); // Setup initial parameter values - arg "init" // arg is either filename or init radius value @@ -264,18 +266,6 @@ int command(int argc, const char *argv[]) { save_iterations, refresh, interrupt, logger, init_writer, sample_writers[0], diagnostic_json_writers[0]); } else { - std::string output_file - = get_arg_val(parser, "output", "file"); - auto base_sfx = get_basename_suffix(output_file); - if (base_sfx.second.empty()) - base_sfx.second = ".csv"; - auto ofs = std::make_unique(base_sfx.first + "_pathfinder" - + base_sfx.second); - if (sig_figs > -1) { - ofs->precision(sig_figs); - } - stan::callbacks::unique_stream_writer pathfinder_writer( - std::move(ofs), "# "); write_stan(pathfinder_writer); write_model(pathfinder_writer, model.model_name()); write_datetime(pathfinder_writer); diff --git a/src/cmdstan/command_helper.hpp b/src/cmdstan/command_helper.hpp index 8948f6a1cb..4573070184 100644 --- a/src/cmdstan/command_helper.hpp +++ b/src/cmdstan/command_helper.hpp @@ -740,6 +740,7 @@ void check_file_config(argument_parser &parser) { } std::vector make_filenames(const std::string &filename, + const std::string &tag, const std::string &type, unsigned int num_chains, unsigned int id) { @@ -756,7 +757,7 @@ std::vector make_filenames(const std::string &filename, } }; for (int i = 0; i < num_chains; ++i) { - names[i] = base_sfx.first + name_iterator(i) + base_sfx.second; + names[i] = base_sfx.first + tag + name_iterator(i) + base_sfx.second; } return names; } @@ -768,23 +769,39 @@ void init_callbacks( std::vector> &diag_csv_writers, std::vector> - &diag_json_writers) { + &diag_json_writers, + stan::callbacks::unique_stream_writer &pathfinder_writer) { + // bookkeeping auto user_method = parser.arg("method"); unsigned int num_chains = get_num_chains(parser); unsigned int id = get_arg_val(parser, "id"); int sig_figs = get_arg_val(parser, "output", "sig_figs"); + bool is_pathfinder = user_method->arg("pathfinder"); + bool save_single_paths = is_pathfinder && num_chains > 1 + && get_arg_val(parser, "method", "pathfinder", "save_single_paths"); + // output.csv files for all chains and single-path-pathfinder csv files sample_writers.reserve(num_chains); - std::vector output_filenames + if (!is_pathfinder || save_single_paths || num_chains == 1) { + std::vector output_filenames = make_filenames(get_arg_val(parser, "output", "file"), - ".csv", num_chains, id); - for (int i = 0; i < num_chains; ++i) { - auto ofs = std::make_unique(output_filenames[i]); - if (sig_figs > -1) - ofs->precision(sig_figs); - sample_writers.emplace_back(std::move(ofs), "# "); + "", ".csv", num_chains, id); + if (save_single_paths) { + output_filenames = make_filenames(get_arg_val(parser, "output", "file"), + "_pathfinder", ".csv", num_chains, id); + } + for (int i = 0; i < num_chains; ++i) { + auto ofs = std::make_unique(output_filenames[i]); + if (sig_figs > -1) + ofs->precision(sig_figs); + sample_writers.emplace_back(std::move(ofs), "# "); + } + } else if (is_pathfinder && !save_single_paths) { + for (int i = 0; i < num_chains; ++i) { + sample_writers.emplace_back(nullptr, "# "); + } } - + // diagnostic files for sampler (csv) and pathfinder (json) diag_json_writers.reserve(num_chains); diag_csv_writers.reserve(num_chains); // create no-op writers by default @@ -802,7 +819,7 @@ void init_callbacks( std::vector diag_filenames; if (user_method->arg("pathfinder")) { diag_json_writers.clear(); - diag_filenames = make_filenames(diagnostic_file, ".json", num_chains, id); + diag_filenames = make_filenames(diagnostic_file, "", ".json", num_chains, id); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(diag_filenames[i]); if (sig_figs > -1) @@ -812,7 +829,7 @@ void init_callbacks( } } else { diag_csv_writers.clear(); - diag_filenames = make_filenames(diagnostic_file, ".csv", num_chains, id); + diag_filenames = make_filenames(diagnostic_file, "", ".csv", num_chains, id); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(diag_filenames[i]); if (sig_figs > -1) @@ -821,6 +838,17 @@ void init_callbacks( } } } + // CSV file for pathfinder PSIS sample + if (is_pathfinder && num_chains > 1) { + std::vector output_filenames = + make_filenames(get_arg_val(parser, "output", "file"), "", ".csv", 1, id); + auto ofs = std::make_unique(output_filenames[0]); + if (sig_figs > -1) + ofs->precision(sig_figs); + stan::callbacks::unique_stream_writer tmp(std::move(ofs), "# "); + pathfinder_writer = std::move(tmp); + } + } } // namespace cmdstan From 55edf120b7ea2ced36045d3a00bc129133c71995 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 17 Aug 2023 16:04:56 -0400 Subject: [PATCH 02/13] unit tests --- src/cmdstan/command.hpp | 9 +- src/cmdstan/command_helper.hpp | 16 +--- src/test/interface/pathfinder_test.cpp | 110 +++++++++++++++++-------- src/test/utility.hpp | 9 ++ 4 files changed, 95 insertions(+), 49 deletions(-) diff --git a/src/cmdstan/command.hpp b/src/cmdstan/command.hpp index 0b3f901866..92143aa558 100644 --- a/src/cmdstan/command.hpp +++ b/src/cmdstan/command.hpp @@ -195,10 +195,9 @@ int command(int argc, const char *argv[]) { diagnostic_csv_writers; std::vector> diagnostic_json_writers; - stan::callbacks::unique_stream_writer pathfinder_writer; init_callbacks(parser, sample_writers, diagnostic_csv_writers, - diagnostic_json_writers, pathfinder_writer); + diagnostic_json_writers); // Setup initial parameter values - arg "init" // arg is either filename or init radius value @@ -266,6 +265,12 @@ int command(int argc, const char *argv[]) { save_iterations, refresh, interrupt, logger, init_writer, sample_writers[0], diagnostic_json_writers[0]); } else { + std::vector output_filenames = + make_filenames(get_arg_val(parser, "output", "file"), "", ".csv", 1, id); + auto ofs = std::make_unique(output_filenames[0]); + if (sig_figs > -1) + ofs->precision(sig_figs); + stan::callbacks::unique_stream_writer pathfinder_writer(std::move(ofs), "# "); write_stan(pathfinder_writer); write_model(pathfinder_writer, model.model_name()); write_datetime(pathfinder_writer); diff --git a/src/cmdstan/command_helper.hpp b/src/cmdstan/command_helper.hpp index 4573070184..1f321e1553 100644 --- a/src/cmdstan/command_helper.hpp +++ b/src/cmdstan/command_helper.hpp @@ -769,8 +769,7 @@ void init_callbacks( std::vector> &diag_csv_writers, std::vector> - &diag_json_writers, - stan::callbacks::unique_stream_writer &pathfinder_writer) { + &diag_json_writers) { // bookkeeping auto user_method = parser.arg("method"); unsigned int num_chains = get_num_chains(parser); @@ -788,7 +787,7 @@ void init_callbacks( "", ".csv", num_chains, id); if (save_single_paths) { output_filenames = make_filenames(get_arg_val(parser, "output", "file"), - "_pathfinder", ".csv", num_chains, id); + "_path", ".csv", num_chains, id); } for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(output_filenames[i]); @@ -838,17 +837,6 @@ void init_callbacks( } } } - // CSV file for pathfinder PSIS sample - if (is_pathfinder && num_chains > 1) { - std::vector output_filenames = - make_filenames(get_arg_val(parser, "output", "file"), "", ".csv", 1, id); - auto ofs = std::make_unique(output_filenames[0]); - if (sig_figs > -1) - ofs->precision(sig_figs); - stan::callbacks::unique_stream_writer tmp(std::move(ofs), "# "); - pathfinder_writer = std::move(tmp); - } - } } // namespace cmdstan diff --git a/src/test/interface/pathfinder_test.cpp b/src/test/interface/pathfinder_test.cpp index 819bcabe82..5faba2bbaa 100644 --- a/src/test/interface/pathfinder_test.cpp +++ b/src/test/interface/pathfinder_test.cpp @@ -1,9 +1,11 @@ #include #include +#include #include using cmdstan::test::convert_model_path; using cmdstan::test::count_matches; +using cmdstan::test::file_exists; using cmdstan::test::parse_sample; using cmdstan::test::run_command; using cmdstan::test::run_command_output; @@ -16,32 +18,39 @@ class CmdStan : public testing::Test { eight_schools_model = {"src", "test", "test-models", "eight_schools"}; eight_schools_data = {"src", "test", "test-models", "eight_schools.data.json"}; - test_arg_output = {"test", "tmp_pf"}; - test_arg_diags = {"test", "tmp_pf"}; - test_result_draws = {"test", "tmp_pf_pathfinder.csv"}; - test_result_single = {"test", "tmp_pf_1.csv"}; - test_result_diags = {"test", "tmp_pf_1.json"}; + arg_output = {"test", "output"}; + arg_diags = {"test", "output"}; + output_csv = {"test", "output.csv"}; + output_save_single = {"test", "output_path_1.csv"}; + output_diags = {"test", "output_1.json"}; } + + void TearDown() { + std::remove(convert_model_path(output_csv).c_str()); + std::remove(convert_model_path(output_save_single).c_str()); + std::remove(convert_model_path(output_diags).c_str()); + } + std::vector dev_null_path; std::vector multi_normal_model; std::vector eight_schools_model; std::vector eight_schools_data; - std::vector test_arg_output; - std::vector test_arg_diags; - std::vector test_result_draws; - std::vector test_result_single; - std::vector test_result_diags; + std::vector arg_output; + std::vector arg_diags; + std::vector output_csv; + std::vector output_save_single; + std::vector output_diags; }; -TEST_F(CmdStan, pathfinder_good) { +TEST_F(CmdStan, pathfinder_40_draws) { std::stringstream ss; ss << convert_model_path(multi_normal_model) - << " output refresh=0 file=" << convert_model_path(test_arg_output) + << " output refresh=0 file=" << convert_model_path(arg_output) << " method=pathfinder num_psis_draws=40"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); - std::fstream result_stream(convert_model_path(test_result_draws)); + std::fstream result_stream(convert_model_path(output_csv)); std::stringstream result_sstream; result_sstream << result_stream.rdbuf(); result_stream.close(); @@ -50,65 +59,100 @@ TEST_F(CmdStan, pathfinder_good) { EXPECT_EQ(1, count_matches(" seconds (Pathfinders)", output)); EXPECT_EQ(1, count_matches(" seconds (PSIS)", output)); EXPECT_EQ(1, count_matches(" seconds (Total)", output)); - EXPECT_EQ(1, count_matches(" seconds (Total)", output)); EXPECT_EQ(1, count_matches("num_psis_draws = 40", output)); + EXPECT_EQ(1, count_matches("num_paths = 4 (Default)", output)); + EXPECT_EQ(1, count_matches("save_single_paths = 0 (Default)", output)); +} - result_sstream.str(std::string()); - std::fstream single_stream(convert_model_path(test_result_single)); +TEST_F(CmdStan, pathfinder_save_single_paths) { + std::stringstream ss; + ss << convert_model_path(multi_normal_model) + << " output refresh=0 file=" << convert_model_path(arg_output) + << " method=pathfinder save_single_paths=1"; + run_command_output out = run_command(ss.str()); + ASSERT_FALSE(out.hasError); + std::fstream single_stream(convert_model_path(output_save_single)); + std::stringstream result_sstream; result_sstream << single_stream.rdbuf(); single_stream.close(); - output = result_sstream.str(); + std::string output = result_sstream.str(); EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", output)); + EXPECT_EQ(1, count_matches("save_single_paths = 1", output)); } TEST_F(CmdStan, pathfinder_single_good) { std::stringstream ss; ss << convert_model_path(multi_normal_model) - << " output refresh=0 file=" << convert_model_path(test_arg_output) + << " output refresh=0 file=" << convert_model_path(arg_output) << " method=pathfinder" << " num_paths=1"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); - std::vector test_result_1path = {"test", "tmp_pf.csv"}; - std::fstream result_stream(convert_model_path(test_result_1path)); + std::fstream result_stream(convert_model_path(output_csv)); std::stringstream result_sstream; result_sstream << result_stream.rdbuf(); result_stream.close(); std::string output = result_sstream.str(); EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", output)); + EXPECT_EQ(1, count_matches("num_paths = 1", output)); + EXPECT_EQ(1, count_matches("save_single_paths = 0 (Default)", output)); } -TEST_F(CmdStan, pathfinder_multi_good) { +TEST_F(CmdStan, pathfinder_single_good_plus) { std::stringstream ss; ss << convert_model_path(multi_normal_model) - << " output refresh=0 file=" << convert_model_path(test_arg_output) + << " output refresh=0 file=" << convert_model_path(arg_output) << " method=pathfinder" - << " num_paths=8"; + << " num_paths=1" + << " save_single_paths=1"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); - std::vector test_result_8path = {"test", "tmp_pf_8.csv"}; - std::fstream result_stream(convert_model_path(test_result_8path)); + + std::fstream result_stream(convert_model_path(output_csv)); std::stringstream result_sstream; result_sstream << result_stream.rdbuf(); result_stream.close(); std::string output = result_sstream.str(); EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", output)); + EXPECT_EQ(1, count_matches("num_paths = 1", output)); + EXPECT_EQ(1, count_matches("save_single_paths = 1", output)); + + ASSERT_FALSE(file_exists(convert_model_path(output_save_single))); +} + +TEST_F(CmdStan, pathfinder_num_paths_8) { + std::stringstream ss; + ss << convert_model_path(multi_normal_model) + << " output refresh=0 file=" << convert_model_path(arg_output) + << " method=pathfinder" + << " num_paths=8"; + run_command_output out = run_command(ss.str()); + ASSERT_FALSE(out.hasError); + std::vector test_psis_path = {"test", "output.csv"}; + std::fstream result_stream(convert_model_path(test_psis_path)); + std::stringstream result_sstream; + result_sstream << result_stream.rdbuf(); + result_stream.close(); + std::string output = result_sstream.str(); + EXPECT_EQ(1, count_matches("num_paths = 8", output)); + EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); + EXPECT_EQ(1, count_matches(" seconds (Pathfinders)", output)); } TEST_F(CmdStan, pathfinder_diagnostic_json) { std::stringstream ss; ss << convert_model_path(multi_normal_model) - << " output refresh=0 file=" << convert_model_path(test_arg_output) - << " diagnostic_file=" << convert_model_path(test_arg_diags) + << " output refresh=0 file=" << convert_model_path(arg_output) + << " diagnostic_file=" << convert_model_path(arg_diags) << " method=pathfinder"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); - std::fstream result_stream(convert_model_path(test_result_diags)); + std::fstream result_stream(convert_model_path(output_diags)); std::stringstream result_sstream; result_sstream << result_stream.rdbuf(); result_stream.close(); @@ -123,13 +167,13 @@ TEST_F(CmdStan, pathfinder_lbfgs_iterations) { ss << convert_model_path(eight_schools_model) << " data file=" << convert_model_path(eight_schools_data) << " random seed=12345" - << " output refresh=0 file=" << convert_model_path(test_arg_output) - << " diagnostic_file=" << convert_model_path(test_arg_diags) + << " output refresh=0 file=" << convert_model_path(arg_output) + << " diagnostic_file=" << convert_model_path(arg_diags) << " method=pathfinder max_lbfgs_iters=3"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); - std::fstream result_stream(convert_model_path(test_result_diags)); + std::fstream result_stream(convert_model_path(output_diags)); std::stringstream result_sstream; result_sstream << result_stream.rdbuf(); result_stream.close(); @@ -145,12 +189,12 @@ TEST_F(CmdStan, pathfinder_num_paths_draws) { std::stringstream ss; ss << convert_model_path(eight_schools_model) << " data file=" << convert_model_path(eight_schools_data) - << " output refresh=0 file=" << convert_model_path(test_arg_output) + << " output refresh=0 file=" << convert_model_path(arg_output) << " method=pathfinder num_draws=10 num_paths=2"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); - std::fstream result_stream(convert_model_path(test_result_draws)); + std::fstream result_stream(convert_model_path(output_csv)); std::stringstream result_sstream; result_sstream << result_stream.rdbuf(); result_stream.close(); diff --git a/src/test/utility.hpp b/src/test/utility.hpp index 7bd6834a46..d76f39c1e4 100644 --- a/src/test/utility.hpp +++ b/src/test/utility.hpp @@ -9,6 +9,9 @@ #include #include +#include + + namespace cmdstan { namespace test { @@ -281,6 +284,12 @@ int idx_first_match(const std::vector &lines, return idx; } +bool file_exists(const std::string& filename) { + struct stat buffer; + return (stat(filename.c_str(), &buffer) == 0); +} + + } // namespace test } // namespace cmdstan #endif From 3d22daf0ca7d423db485ce3436b722e6a646ed1b Mon Sep 17 00:00:00 2001 From: Stan Jenkins Date: Thu, 17 Aug 2023 16:32:42 -0400 Subject: [PATCH 03/13] [Jenkins] auto-formatting by clang-format version 10.0.0-4ubuntu1 --- src/cmdstan/arguments/arg_pathfinder.hpp | 5 +++-- src/cmdstan/command.hpp | 8 +++++--- src/cmdstan/command_helper.hpp | 23 ++++++++++++++--------- src/test/utility.hpp | 8 +++----- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/cmdstan/arguments/arg_pathfinder.hpp b/src/cmdstan/arguments/arg_pathfinder.hpp index 09d1795ddc..b5b0671edb 100644 --- a/src/cmdstan/arguments/arg_pathfinder.hpp +++ b/src/cmdstan/arguments/arg_pathfinder.hpp @@ -16,8 +16,9 @@ class arg_pathfinder : public arg_lbfgs { _subarguments.push_back(new arg_single_int_pos( "num_psis_draws", "Number of draws from PSIS sample", 1000)); - _subarguments.push_back( - new arg_single_bool("save_single_paths", "Output single-path pathfinder draws as CSV", false)); + _subarguments.push_back(new arg_single_bool( + "save_single_paths", "Output single-path pathfinder draws as CSV", + false)); _subarguments.push_back( new arg_single_int_pos("num_paths", "Number of single pathfinders", 4)); _subarguments.push_back(new arg_single_int_pos( diff --git a/src/cmdstan/command.hpp b/src/cmdstan/command.hpp index 92143aa558..57270da279 100644 --- a/src/cmdstan/command.hpp +++ b/src/cmdstan/command.hpp @@ -265,12 +265,14 @@ int command(int argc, const char *argv[]) { save_iterations, refresh, interrupt, logger, init_writer, sample_writers[0], diagnostic_json_writers[0]); } else { - std::vector output_filenames = - make_filenames(get_arg_val(parser, "output", "file"), "", ".csv", 1, id); + std::vector output_filenames = make_filenames( + get_arg_val(parser, "output", "file"), "", ".csv", 1, + id); auto ofs = std::make_unique(output_filenames[0]); if (sig_figs > -1) ofs->precision(sig_figs); - stan::callbacks::unique_stream_writer pathfinder_writer(std::move(ofs), "# "); + stan::callbacks::unique_stream_writer pathfinder_writer( + std::move(ofs), "# "); write_stan(pathfinder_writer); write_model(pathfinder_writer, model.model_name()); write_datetime(pathfinder_writer); diff --git a/src/cmdstan/command_helper.hpp b/src/cmdstan/command_helper.hpp index 38b6d0904e..c8839de0f2 100644 --- a/src/cmdstan/command_helper.hpp +++ b/src/cmdstan/command_helper.hpp @@ -767,25 +767,28 @@ void init_callbacks( std::vector> &diag_csv_writers, std::vector> - &diag_json_writers) { + &diag_json_writers) { // bookkeeping auto user_method = parser.arg("method"); unsigned int num_chains = get_num_chains(parser); unsigned int id = get_arg_val(parser, "id"); int sig_figs = get_arg_val(parser, "output", "sig_figs"); bool is_pathfinder = user_method->arg("pathfinder"); - bool save_single_paths = is_pathfinder && num_chains > 1 - && get_arg_val(parser, "method", "pathfinder", "save_single_paths"); + bool save_single_paths + = is_pathfinder && num_chains > 1 + && get_arg_val(parser, "method", "pathfinder", + "save_single_paths"); // output.csv files for all chains and single-path-pathfinder csv files sample_writers.reserve(num_chains); if (!is_pathfinder || save_single_paths || num_chains == 1) { std::vector output_filenames - = make_filenames(get_arg_val(parser, "output", "file"), - "", ".csv", num_chains, id); + = make_filenames(get_arg_val(parser, "output", "file"), + "", ".csv", num_chains, id); if (save_single_paths) { - output_filenames = make_filenames(get_arg_val(parser, "output", "file"), - "_path", ".csv", num_chains, id); + output_filenames = make_filenames( + get_arg_val(parser, "output", "file"), "_path", + ".csv", num_chains, id); } for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(output_filenames[i]); @@ -816,7 +819,8 @@ void init_callbacks( std::vector diag_filenames; if (user_method->arg("pathfinder")) { diag_json_writers.clear(); - diag_filenames = make_filenames(diagnostic_file, "", ".json", num_chains, id); + diag_filenames + = make_filenames(diagnostic_file, "", ".json", num_chains, id); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(diag_filenames[i]); if (sig_figs > -1) @@ -826,7 +830,8 @@ void init_callbacks( } } else { diag_csv_writers.clear(); - diag_filenames = make_filenames(diagnostic_file, "", ".csv", num_chains, id); + diag_filenames + = make_filenames(diagnostic_file, "", ".csv", num_chains, id); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(diag_filenames[i]); if (sig_figs > -1) diff --git a/src/test/utility.hpp b/src/test/utility.hpp index d76f39c1e4..004e9e516d 100644 --- a/src/test/utility.hpp +++ b/src/test/utility.hpp @@ -11,7 +11,6 @@ #include - namespace cmdstan { namespace test { @@ -284,12 +283,11 @@ int idx_first_match(const std::vector &lines, return idx; } -bool file_exists(const std::string& filename) { - struct stat buffer; - return (stat(filename.c_str(), &buffer) == 0); +bool file_exists(const std::string &filename) { + struct stat buffer; + return (stat(filename.c_str(), &buffer) == 0); } - } // namespace test } // namespace cmdstan #endif From f4a93bef4f75be45c298408907653163bd6812a8 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 18 Aug 2023 17:49:28 -0400 Subject: [PATCH 04/13] checkpointing --- src/cmdstan/command.hpp | 15 +++- src/cmdstan/command_helper.hpp | 107 ++++++++++++---------- src/test/interface/pathfinder_test.cpp | 120 ++++++++++++++----------- 3 files changed, 138 insertions(+), 104 deletions(-) diff --git a/src/cmdstan/command.hpp b/src/cmdstan/command.hpp index 57270da279..7f811c7ce8 100644 --- a/src/cmdstan/command.hpp +++ b/src/cmdstan/command.hpp @@ -208,9 +208,11 @@ int command(int argc, const char *argv[]) { init = ""; } catch (const std::logic_error &e) { } + std::vector> init_contexts = get_vec_var_context(init, num_chains); std::vector model_compile_info = model.model_compile_info(); + for (int i = 0; i < num_chains; ++i) { write_stan(sample_writers[i]); write_model(sample_writers[i], model.model_name()); @@ -254,8 +256,8 @@ int command(int argc, const char *argv[]) { = get_arg_val(*pathfinder_arg, "num_psis_draws"); int num_paths = get_arg_val(*pathfinder_arg, "num_paths"); bool save_iterations - = !get_arg_val(parser, "output", "diagnostic_file") - .empty(); + = get_arg_val(*pathfinder_arg, "save_single_paths"); + if (num_paths == 1) { return_code = stan::services::pathfinder::pathfinder_lbfgs_single< false, stan::model::model_base>( @@ -265,14 +267,19 @@ int command(int argc, const char *argv[]) { save_iterations, refresh, interrupt, logger, init_writer, sample_writers[0], diagnostic_json_writers[0]); } else { - std::vector output_filenames = make_filenames( + std::vector pf_names = make_filenames( get_arg_val(parser, "output", "file"), "", ".csv", 1, id); - auto ofs = std::make_unique(output_filenames[0]); + auto ofs = std::make_unique(pf_names[0]); if (sig_figs > -1) ofs->precision(sig_figs); stan::callbacks::unique_stream_writer pathfinder_writer( std::move(ofs), "# "); + std::cout << "L 280" << std::endl; + pathfinder_writer("# test"); + for (int i = 0; i < num_paths; ++i) { + diagnostic_json_writers[i].write("test", "test"); + } write_stan(pathfinder_writer); write_model(pathfinder_writer, model.model_name()); write_datetime(pathfinder_writer); diff --git a/src/cmdstan/command_helper.hpp b/src/cmdstan/command_helper.hpp index c8839de0f2..361c232748 100644 --- a/src/cmdstan/command_helper.hpp +++ b/src/cmdstan/command_helper.hpp @@ -744,9 +744,7 @@ std::vector make_filenames(const std::string &filename, unsigned int id) { std::vector names(num_chains); auto base_sfx = get_basename_suffix(filename); - if (base_sfx.second.empty()) { - base_sfx.second = type; - } + base_sfx.second = type; auto name_iterator = [num_chains, id](auto i) { if (num_chains == 1) { return std::string(""); @@ -768,72 +766,85 @@ void init_callbacks( &diag_csv_writers, std::vector> &diag_json_writers) { - // bookkeeping auto user_method = parser.arg("method"); unsigned int num_chains = get_num_chains(parser); unsigned int id = get_arg_val(parser, "id"); int sig_figs = get_arg_val(parser, "output", "sig_figs"); bool is_pathfinder = user_method->arg("pathfinder"); - bool save_single_paths - = is_pathfinder && num_chains > 1 + bool save_single_paths = is_pathfinder && get_arg_val(parser, "method", "pathfinder", "save_single_paths"); - // output.csv files for all chains and single-path-pathfinder csv files + std::vector output_filenames; + std::vector diagnostic_filenames; + sample_writers.reserve(num_chains); - if (!is_pathfinder || save_single_paths || num_chains == 1) { - std::vector output_filenames - = make_filenames(get_arg_val(parser, "output", "file"), - "", ".csv", num_chains, id); - if (save_single_paths) { + diag_csv_writers.reserve(num_chains); + diag_json_writers.reserve(num_chains); + + // create no-op diagnostics writers by default + for (int i = 0; i < num_chains; ++i) { + diag_csv_writers.emplace_back(nullptr, "# "); + diag_json_writers.emplace_back( + stan::callbacks::json_writer()); + } + // sample_writers for pathfinder - if saving or only single-path run + if (is_pathfinder && (!save_single_paths || num_chains > 1)) { + for (int i = 0; i < num_chains; ++i) { + sample_writers.emplace_back(nullptr, "# "); + } + } + + if (save_single_paths) { + sample_writers.clear(); + diag_json_writers.clear(); + if (num_chains == 1) { // no multi-path outputs, don't need sfx "_path" output_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "_path", - ".csv", num_chains, id); + get_arg_val(parser, "output", "file"), + "", ".csv", num_chains, id); + diagnostic_filenames = make_filenames( + get_arg_val(parser, "output", "file"), + "", ".json", num_chains, id); + } else { // distinguish single, multi-path output files + output_filenames = make_filenames( + get_arg_val(parser, "output", "file"), + "_path", ".csv", num_chains, id); + diagnostic_filenames = make_filenames( + get_arg_val(parser, "output", "file"), + "_path", ".json", num_chains, id); } for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(output_filenames[i]); - if (sig_figs > -1) + auto ofs_diag = std::make_unique(diagnostic_filenames[i]); + if (sig_figs > -1) { ofs->precision(sig_figs); + ofs_diag->precision(sig_figs); + } sample_writers.emplace_back(std::move(ofs), "# "); + stan::callbacks::json_writer jwriter(std::move(ofs_diag)); + diag_json_writers.emplace_back(std::move(jwriter)); } - } else if (is_pathfinder && !save_single_paths) { + } else if (!is_pathfinder || num_chains == 1) { + output_filenames = make_filenames( + get_arg_val(parser, "output", "file"), + "", ".csv", num_chains, id); for (int i = 0; i < num_chains; ++i) { - sample_writers.emplace_back(nullptr, "# "); + auto ofs = std::make_unique(output_filenames[i]); + if (sig_figs > -1) + ofs->precision(sig_figs); + sample_writers.emplace_back(std::move(ofs), "# "); } } - // diagnostic files for sampler (csv) and pathfinder (json) - diag_json_writers.reserve(num_chains); - diag_csv_writers.reserve(num_chains); - // create no-op writers by default - for (int i = 0; i < num_chains; ++i) { - diag_json_writers.emplace_back( - stan::callbacks::json_writer()); - } - for (int i = 0; i < num_chains; ++i) { - diag_csv_writers.emplace_back(nullptr, "# "); - } - // create json, csv writers as needed. - std::string diagnostic_file - = get_arg_val(parser, "output", "diagnostic_file"); - if (!diagnostic_file.empty()) { - std::vector diag_filenames; - if (user_method->arg("pathfinder")) { - diag_json_writers.clear(); - diag_filenames - = make_filenames(diagnostic_file, "", ".json", num_chains, id); - for (int i = 0; i < num_chains; ++i) { - auto ofs = std::make_unique(diag_filenames[i]); - if (sig_figs > -1) - ofs->precision(sig_figs); - stan::callbacks::json_writer jwriter(std::move(ofs)); - diag_json_writers.emplace_back(std::move(jwriter)); - } - } else { + + if (!is_pathfinder) { + std::string diagnostic_file + = get_arg_val(parser, "output", "diagnostic_file"); + if (!diagnostic_file.empty()) { diag_csv_writers.clear(); - diag_filenames - = make_filenames(diagnostic_file, "", ".csv", num_chains, id); + diagnostic_filenames = make_filenames(diagnostic_file, + "", ".csv", num_chains, id); for (int i = 0; i < num_chains; ++i) { - auto ofs = std::make_unique(diag_filenames[i]); + auto ofs = std::make_unique(diagnostic_filenames[i]); if (sig_figs > -1) ofs->precision(sig_figs); diag_csv_writers.emplace_back(std::move(ofs), "# "); diff --git a/src/test/interface/pathfinder_test.cpp b/src/test/interface/pathfinder_test.cpp index 5faba2bbaa..fe77f57976 100644 --- a/src/test/interface/pathfinder_test.cpp +++ b/src/test/interface/pathfinder_test.cpp @@ -22,13 +22,16 @@ class CmdStan : public testing::Test { arg_diags = {"test", "output"}; output_csv = {"test", "output.csv"}; output_save_single = {"test", "output_path_1.csv"}; - output_diags = {"test", "output_1.json"}; + output_save_single_diag = {"test", "output_path_1.json"}; + output_save_single_diag_1 = {"test", "output.json"}; + } void TearDown() { std::remove(convert_model_path(output_csv).c_str()); std::remove(convert_model_path(output_save_single).c_str()); - std::remove(convert_model_path(output_diags).c_str()); + std::remove(convert_model_path(output_save_single_diag).c_str()); + std::remove(convert_model_path(output_save_single_diag_1).c_str()); } std::vector dev_null_path; @@ -39,7 +42,8 @@ class CmdStan : public testing::Test { std::vector arg_diags; std::vector output_csv; std::vector output_save_single; - std::vector output_diags; + std::vector output_save_single_diag; + std::vector output_save_single_diag_1; }; TEST_F(CmdStan, pathfinder_40_draws) { @@ -71,17 +75,28 @@ TEST_F(CmdStan, pathfinder_save_single_paths) { << " method=pathfinder save_single_paths=1"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); - std::fstream single_stream(convert_model_path(output_save_single)); + std::fstream single_csv_stream(convert_model_path(output_save_single)); std::stringstream result_sstream; - result_sstream << single_stream.rdbuf(); - single_stream.close(); - std::string output = result_sstream.str(); - EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); - EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", output)); - EXPECT_EQ(1, count_matches("save_single_paths = 1", output)); + result_sstream << single_csv_stream.rdbuf(); + single_csv_stream.close(); + std::string output_csv = result_sstream.str(); + EXPECT_EQ(1, count_matches("# Elapsed Time:", output_csv)); + EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", output_csv)); + EXPECT_EQ(1, count_matches("save_single_paths = 1", output_csv)); + + std::fstream single_json_stream(convert_model_path(output_save_single_diag)); + std::stringstream result_json_sstream; + result_json_sstream << single_json_stream.rdbuf(); + single_json_stream.close(); + std::string output_json = result_json_sstream.str(); + ASSERT_FALSE(output_json.empty()); + + rapidjson::Document document; + ASSERT_FALSE(document.Parse<0>(output_json.c_str()).HasParseError()); + EXPECT_EQ(1, count_matches("\"1\" : {\"iter\" : 1,", output_json)); } -TEST_F(CmdStan, pathfinder_single_good) { +TEST_F(CmdStan, pathfinder_single) { std::stringstream ss; ss << convert_model_path(multi_normal_model) << " output refresh=0 file=" << convert_model_path(arg_output) @@ -143,47 +158,48 @@ TEST_F(CmdStan, pathfinder_num_paths_8) { EXPECT_EQ(1, count_matches(" seconds (Pathfinders)", output)); } -TEST_F(CmdStan, pathfinder_diagnostic_json) { - std::stringstream ss; - ss << convert_model_path(multi_normal_model) - << " output refresh=0 file=" << convert_model_path(arg_output) - << " diagnostic_file=" << convert_model_path(arg_diags) - << " method=pathfinder"; - run_command_output out = run_command(ss.str()); - ASSERT_FALSE(out.hasError); - - std::fstream result_stream(convert_model_path(output_diags)); - std::stringstream result_sstream; - result_sstream << result_stream.rdbuf(); - result_stream.close(); - std::string output = result_sstream.str(); - ASSERT_FALSE(output.empty()); - rapidjson::Document document; - ASSERT_FALSE(document.Parse<0>(output.c_str()).HasParseError()); -} - -TEST_F(CmdStan, pathfinder_lbfgs_iterations) { - std::stringstream ss; - ss << convert_model_path(eight_schools_model) - << " data file=" << convert_model_path(eight_schools_data) - << " random seed=12345" - << " output refresh=0 file=" << convert_model_path(arg_output) - << " diagnostic_file=" << convert_model_path(arg_diags) - << " method=pathfinder max_lbfgs_iters=3"; - run_command_output out = run_command(ss.str()); - ASSERT_FALSE(out.hasError); - - std::fstream result_stream(convert_model_path(output_diags)); - std::stringstream result_sstream; - result_sstream << result_stream.rdbuf(); - result_stream.close(); - std::string output = result_sstream.str(); - ASSERT_FALSE(output.empty()); - rapidjson::Document document; - ASSERT_FALSE(document.Parse<0>(output.c_str()).HasParseError()); - EXPECT_EQ(1, count_matches("\"3\" : {\"iter\" : 3,", output)); - EXPECT_EQ(0, count_matches("\"4\" : {\"iter\" : 4,", output)); -} +// TEST_F(CmdStan, pathfinder_diagnostic_json) { +// std::stringstream ss; +// ss << convert_model_path(multi_normal_model) +// << " output refresh=0 file=" << convert_model_path(arg_output) +// << " diagnostic_file=" << convert_model_path(arg_diags) +// << " method=pathfinder"; +// run_command_output out = run_command(ss.str()); +// ASSERT_FALSE(out.hasError); + +// std::fstream result_stream(convert_model_path(output_diags)); +// std::stringstream result_sstream; +// result_sstream << result_stream.rdbuf(); +// result_stream.close(); +// std::string output = result_sstream.str(); +// ASSERT_FALSE(output.empty()); +// rapidjson::Document document; +// ASSERT_FALSE(document.Parse<0>(output.c_str()).HasParseError()); +// } + + +// TEST_F(CmdStan, pathfinder_lbfgs_iterations) { +// std::stringstream ss; +// ss << convert_model_path(eight_schools_model) +// << " data file=" << convert_model_path(eight_schools_data) +// << " random seed=12345" +// << " output refresh=0 file=" << convert_model_path(arg_output) +// << " diagnostic_file=" << convert_model_path(arg_diags) +// << " method=pathfinder max_lbfgs_iters=3"; +// run_command_output out = run_command(ss.str()); +// ASSERT_FALSE(out.hasError); + +// std::fstream result_stream(convert_model_path(output_diags)); +// std::stringstream result_sstream; +// result_sstream << result_stream.rdbuf(); +// result_stream.close(); +// std::string output = result_sstream.str(); +// ASSERT_FALSE(output.empty()); +// rapidjson::Document document; +// ASSERT_FALSE(document.Parse<0>(output.c_str()).HasParseError()); +// EXPECT_EQ(1, count_matches("\"3\" : {\"iter\" : 3,", output)); +// EXPECT_EQ(0, count_matches("\"4\" : {\"iter\" : 4,", output)); +// } TEST_F(CmdStan, pathfinder_num_paths_draws) { std::stringstream ss; From e0d6f37878e8528c7289bef388cb31cbe4cfdb89 Mon Sep 17 00:00:00 2001 From: Stan Jenkins Date: Fri, 18 Aug 2023 17:53:29 -0400 Subject: [PATCH 05/13] [Jenkins] auto-formatting by clang-format version 10.0.0-4ubuntu1 --- src/cmdstan/command.hpp | 2 +- src/cmdstan/command_helper.hpp | 29 +++++++++++++------------- src/test/interface/pathfinder_test.cpp | 2 -- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/cmdstan/command.hpp b/src/cmdstan/command.hpp index 7f811c7ce8..2c8ddaadd1 100644 --- a/src/cmdstan/command.hpp +++ b/src/cmdstan/command.hpp @@ -279,7 +279,7 @@ int command(int argc, const char *argv[]) { pathfinder_writer("# test"); for (int i = 0; i < num_paths; ++i) { diagnostic_json_writers[i].write("test", "test"); - } + } write_stan(pathfinder_writer); write_model(pathfinder_writer, model.model_name()); write_datetime(pathfinder_writer); diff --git a/src/cmdstan/command_helper.hpp b/src/cmdstan/command_helper.hpp index 361c232748..b8b08d0b1e 100644 --- a/src/cmdstan/command_helper.hpp +++ b/src/cmdstan/command_helper.hpp @@ -771,7 +771,8 @@ void init_callbacks( unsigned int id = get_arg_val(parser, "id"); int sig_figs = get_arg_val(parser, "output", "sig_figs"); bool is_pathfinder = user_method->arg("pathfinder"); - bool save_single_paths = is_pathfinder + bool save_single_paths + = is_pathfinder && get_arg_val(parser, "method", "pathfinder", "save_single_paths"); @@ -800,18 +801,18 @@ void init_callbacks( diag_json_writers.clear(); if (num_chains == 1) { // no multi-path outputs, don't need sfx "_path" output_filenames = make_filenames( - get_arg_val(parser, "output", "file"), - "", ".csv", num_chains, id); + get_arg_val(parser, "output", "file"), "", ".csv", + num_chains, id); diagnostic_filenames = make_filenames( - get_arg_val(parser, "output", "file"), - "", ".json", num_chains, id); + get_arg_val(parser, "output", "file"), "", ".json", + num_chains, id); } else { // distinguish single, multi-path output files output_filenames = make_filenames( - get_arg_val(parser, "output", "file"), - "_path", ".csv", num_chains, id); + get_arg_val(parser, "output", "file"), "_path", + ".csv", num_chains, id); diagnostic_filenames = make_filenames( - get_arg_val(parser, "output", "file"), - "_path", ".json", num_chains, id); + get_arg_val(parser, "output", "file"), "_path", + ".json", num_chains, id); } for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(output_filenames[i]); @@ -825,9 +826,9 @@ void init_callbacks( diag_json_writers.emplace_back(std::move(jwriter)); } } else if (!is_pathfinder || num_chains == 1) { - output_filenames = make_filenames( - get_arg_val(parser, "output", "file"), - "", ".csv", num_chains, id); + output_filenames + = make_filenames(get_arg_val(parser, "output", "file"), + "", ".csv", num_chains, id); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(output_filenames[i]); if (sig_figs > -1) @@ -841,8 +842,8 @@ void init_callbacks( = get_arg_val(parser, "output", "diagnostic_file"); if (!diagnostic_file.empty()) { diag_csv_writers.clear(); - diagnostic_filenames = make_filenames(diagnostic_file, - "", ".csv", num_chains, id); + diagnostic_filenames + = make_filenames(diagnostic_file, "", ".csv", num_chains, id); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(diagnostic_filenames[i]); if (sig_figs > -1) diff --git a/src/test/interface/pathfinder_test.cpp b/src/test/interface/pathfinder_test.cpp index fe77f57976..9fd3a23396 100644 --- a/src/test/interface/pathfinder_test.cpp +++ b/src/test/interface/pathfinder_test.cpp @@ -24,7 +24,6 @@ class CmdStan : public testing::Test { output_save_single = {"test", "output_path_1.csv"}; output_save_single_diag = {"test", "output_path_1.json"}; output_save_single_diag_1 = {"test", "output.json"}; - } void TearDown() { @@ -177,7 +176,6 @@ TEST_F(CmdStan, pathfinder_num_paths_8) { // ASSERT_FALSE(document.Parse<0>(output.c_str()).HasParseError()); // } - // TEST_F(CmdStan, pathfinder_lbfgs_iterations) { // std::stringstream ss; // ss << convert_model_path(eight_schools_model) From fc279a4d3dc1fda1a3d4f7abfad0ba61c236b9ec Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Sat, 19 Aug 2023 17:48:15 -0400 Subject: [PATCH 06/13] checkpointing --- src/cmdstan/command_helper.hpp | 61 +++++++++++++++++----------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/cmdstan/command_helper.hpp b/src/cmdstan/command_helper.hpp index b8b08d0b1e..f6a37a146b 100644 --- a/src/cmdstan/command_helper.hpp +++ b/src/cmdstan/command_helper.hpp @@ -771,11 +771,11 @@ void init_callbacks( unsigned int id = get_arg_val(parser, "id"); int sig_figs = get_arg_val(parser, "output", "sig_figs"); bool is_pathfinder = user_method->arg("pathfinder"); - bool save_single_paths - = is_pathfinder - && get_arg_val(parser, "method", "pathfinder", - "save_single_paths"); - + bool save_single_paths = is_pathfinder + && get_arg_val(parser, "method", "pathfinder", + "save_single_paths"); + std::string diagnostic_file + = get_arg_val(parser, "output", "diagnostic_file"); std::vector output_filenames; std::vector diagnostic_filenames; @@ -783,30 +783,29 @@ void init_callbacks( diag_csv_writers.reserve(num_chains); diag_json_writers.reserve(num_chains); - // create no-op diagnostics writers by default for (int i = 0; i < num_chains; ++i) { diag_csv_writers.emplace_back(nullptr, "# "); diag_json_writers.emplace_back( stan::callbacks::json_writer()); } - // sample_writers for pathfinder - if saving or only single-path run - if (is_pathfinder && (!save_single_paths || num_chains > 1)) { - for (int i = 0; i < num_chains; ++i) { - sample_writers.emplace_back(nullptr, "# "); - } - } - if (save_single_paths) { - sample_writers.clear(); - diag_json_writers.clear(); - if (num_chains == 1) { // no multi-path outputs, don't need sfx "_path" + if (is_pathfinder) { + if (num_chains == 1) { output_filenames = make_filenames( get_arg_val(parser, "output", "file"), "", ".csv", num_chains, id); - diagnostic_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "", ".json", - num_chains, id); - } else { // distinguish single, multi-path output files + if (!diagnostic_file.empty()) { + diagnostic_filenames + = make_filenames(get_arg_val(parser, "output", + "diagnostic_file"), + "", ".json", num_chains, id); + } else if (save_single_paths) { + diagnostic_filenames + = make_filenames(get_arg_val(parser, "output", + "file"), + "", ".json", num_chains, id); + } + } else { // add to filename to distinguish single, multi-path outputs output_filenames = make_filenames( get_arg_val(parser, "output", "file"), "_path", ".csv", num_chains, id); @@ -814,18 +813,25 @@ void init_callbacks( get_arg_val(parser, "output", "file"), "_path", ".json", num_chains, id); } + if (!diagnostic_filenames.empty()) + diag_json_writers.clear(); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(output_filenames[i]); auto ofs_diag = std::make_unique(diagnostic_filenames[i]); if (sig_figs > -1) { ofs->precision(sig_figs); - ofs_diag->precision(sig_figs); } sample_writers.emplace_back(std::move(ofs), "# "); - stan::callbacks::json_writer jwriter(std::move(ofs_diag)); - diag_json_writers.emplace_back(std::move(jwriter)); + if (!diagnostic_filenames.empty()) { + auto ofs_diag = std::make_unique(diagnostic_filenames[i]); + if (sig_figs > -1) { + ofs_diag->precision(sig_figs); + } + stan::callbacks::json_writer jwriter(std::move(ofs_diag)); + diag_json_writers.emplace_back(std::move(jwriter)); + } } - } else if (!is_pathfinder || num_chains == 1) { + } else { output_filenames = make_filenames(get_arg_val(parser, "output", "file"), "", ".csv", num_chains, id); @@ -835,15 +841,10 @@ void init_callbacks( ofs->precision(sig_figs); sample_writers.emplace_back(std::move(ofs), "# "); } - } - - if (!is_pathfinder) { - std::string diagnostic_file - = get_arg_val(parser, "output", "diagnostic_file"); if (!diagnostic_file.empty()) { diag_csv_writers.clear(); diagnostic_filenames - = make_filenames(diagnostic_file, "", ".csv", num_chains, id); + = make_filenames(diagnostic_file, "", ".csv", num_chains, id); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(diagnostic_filenames[i]); if (sig_figs > -1) From 76610feada1d3d6b5a98770bebf6d1b2969ede3c Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Sat, 19 Aug 2023 20:03:10 -0400 Subject: [PATCH 07/13] checkpointing --- src/cmdstan/command.hpp | 5 - src/cmdstan/command_helper.hpp | 70 +++++---- src/test/interface/pathfinder_test.cpp | 188 ++++++++++++++----------- 3 files changed, 149 insertions(+), 114 deletions(-) diff --git a/src/cmdstan/command.hpp b/src/cmdstan/command.hpp index 2c8ddaadd1..5bbeedd948 100644 --- a/src/cmdstan/command.hpp +++ b/src/cmdstan/command.hpp @@ -275,11 +275,6 @@ int command(int argc, const char *argv[]) { ofs->precision(sig_figs); stan::callbacks::unique_stream_writer pathfinder_writer( std::move(ofs), "# "); - std::cout << "L 280" << std::endl; - pathfinder_writer("# test"); - for (int i = 0; i < num_paths; ++i) { - diagnostic_json_writers[i].write("test", "test"); - } write_stan(pathfinder_writer); write_model(pathfinder_writer, model.model_name()); write_datetime(pathfinder_writer); diff --git a/src/cmdstan/command_helper.hpp b/src/cmdstan/command_helper.hpp index f6a37a146b..f2fc182696 100644 --- a/src/cmdstan/command_helper.hpp +++ b/src/cmdstan/command_helper.hpp @@ -770,10 +770,8 @@ void init_callbacks( unsigned int num_chains = get_num_chains(parser); unsigned int id = get_arg_val(parser, "id"); int sig_figs = get_arg_val(parser, "output", "sig_figs"); - bool is_pathfinder = user_method->arg("pathfinder"); - bool save_single_paths = is_pathfinder - && get_arg_val(parser, "method", "pathfinder", - "save_single_paths"); + bool save_single_paths = user_method->arg("pathfinder") + && get_arg_val(parser, "method", "pathfinder", "save_single_paths"); std::string diagnostic_file = get_arg_val(parser, "output", "diagnostic_file"); std::vector output_filenames; @@ -783,17 +781,20 @@ void init_callbacks( diag_csv_writers.reserve(num_chains); diag_json_writers.reserve(num_chains); + // default - no diagnostics for (int i = 0; i < num_chains; ++i) { diag_csv_writers.emplace_back(nullptr, "# "); - diag_json_writers.emplace_back( - stan::callbacks::json_writer()); + diag_json_writers.emplace_back(stan::callbacks::json_writer()); } - - if (is_pathfinder) { + + if (user_method->arg("pathfinder")) { + bool inst_writers = true; + bool inst_diags = true; + // make filenames if (num_chains == 1) { - output_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "", ".csv", - num_chains, id); + output_filenames + = make_filenames(get_arg_val(parser, "output", "file"), + "", ".csv", num_chains, id); if (!diagnostic_file.empty()) { diagnostic_filenames = make_filenames(get_arg_val(parser, "output", @@ -804,25 +805,36 @@ void init_callbacks( = make_filenames(get_arg_val(parser, "output", "file"), "", ".json", num_chains, id); + } else { + inst_diags = false; } - } else { // add to filename to distinguish single, multi-path outputs + } else if (save_single_paths) { // filenames for single-path outputs output_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "_path", - ".csv", num_chains, id); + get_arg_val(parser, "output", "file"), "_path", + ".csv", num_chains, id); diagnostic_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "_path", - ".json", num_chains, id); + get_arg_val(parser, "output", "file"), "_path", + ".json", num_chains, id); + } else { // multi-path default: don't save single-path outputs + inst_writers = false; + inst_diags = false; + for (int i = 0; i < num_chains; ++i) { + sample_writers.emplace_back(nullptr, "# "); + } } - if (!diagnostic_filenames.empty()) - diag_json_writers.clear(); - for (int i = 0; i < num_chains; ++i) { - auto ofs = std::make_unique(output_filenames[i]); - auto ofs_diag = std::make_unique(diagnostic_filenames[i]); - if (sig_figs > -1) { - ofs->precision(sig_figs); + // allocate writers + if (inst_writers) { + for (int i = 0; i < num_chains; ++i) { + auto ofs = std::make_unique(output_filenames[i]); + if (sig_figs > -1) { + ofs->precision(sig_figs); + } + sample_writers.emplace_back(std::move(ofs), "# "); } - sample_writers.emplace_back(std::move(ofs), "# "); - if (!diagnostic_filenames.empty()) { + } + if (inst_diags) { + diag_json_writers.clear(); + for (int i = 0; i < num_chains; ++i) { auto ofs_diag = std::make_unique(diagnostic_filenames[i]); if (sig_figs > -1) { ofs_diag->precision(sig_figs); @@ -830,11 +842,11 @@ void init_callbacks( stan::callbacks::json_writer jwriter(std::move(ofs_diag)); diag_json_writers.emplace_back(std::move(jwriter)); } - } - } else { + } + } else { // not pathfinder output_filenames - = make_filenames(get_arg_val(parser, "output", "file"), - "", ".csv", num_chains, id); + = make_filenames(get_arg_val(parser, "output", "file"), + "", ".csv", num_chains, id); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(output_filenames[i]); if (sig_figs > -1) diff --git a/src/test/interface/pathfinder_test.cpp b/src/test/interface/pathfinder_test.cpp index 9fd3a23396..5af56bd6ef 100644 --- a/src/test/interface/pathfinder_test.cpp +++ b/src/test/interface/pathfinder_test.cpp @@ -21,16 +21,16 @@ class CmdStan : public testing::Test { arg_output = {"test", "output"}; arg_diags = {"test", "output"}; output_csv = {"test", "output.csv"}; - output_save_single = {"test", "output_path_1.csv"}; - output_save_single_diag = {"test", "output_path_1.json"}; - output_save_single_diag_1 = {"test", "output.json"}; + output_json = {"test", "output.json"}; + output_single_csv = {"test", "output_path_4.csv"}; + output_single_json = {"test", "output_path_4.json"}; } void TearDown() { std::remove(convert_model_path(output_csv).c_str()); - std::remove(convert_model_path(output_save_single).c_str()); - std::remove(convert_model_path(output_save_single_diag).c_str()); - std::remove(convert_model_path(output_save_single_diag_1).c_str()); + std::remove(convert_model_path(output_json).c_str()); + std::remove(convert_model_path(output_single_csv).c_str()); + std::remove(convert_model_path(output_single_json).c_str()); } std::vector dev_null_path; @@ -40,16 +40,16 @@ class CmdStan : public testing::Test { std::vector arg_output; std::vector arg_diags; std::vector output_csv; - std::vector output_save_single; - std::vector output_save_single_diag; - std::vector output_save_single_diag_1; + std::vector output_json; + std::vector output_single_csv; + std::vector output_single_json; }; -TEST_F(CmdStan, pathfinder_40_draws) { +TEST_F(CmdStan, pathfinder_defaults) { std::stringstream ss; ss << convert_model_path(multi_normal_model) << " output refresh=0 file=" << convert_model_path(arg_output) - << " method=pathfinder num_psis_draws=40"; + << " method=pathfinder"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); @@ -62,45 +62,15 @@ TEST_F(CmdStan, pathfinder_40_draws) { EXPECT_EQ(1, count_matches(" seconds (Pathfinders)", output)); EXPECT_EQ(1, count_matches(" seconds (PSIS)", output)); EXPECT_EQ(1, count_matches(" seconds (Total)", output)); - EXPECT_EQ(1, count_matches("num_psis_draws = 40", output)); - EXPECT_EQ(1, count_matches("num_paths = 4 (Default)", output)); EXPECT_EQ(1, count_matches("save_single_paths = 0 (Default)", output)); + EXPECT_EQ(1, count_matches("num_paths = 4 (Default)", output)); } -TEST_F(CmdStan, pathfinder_save_single_paths) { - std::stringstream ss; - ss << convert_model_path(multi_normal_model) - << " output refresh=0 file=" << convert_model_path(arg_output) - << " method=pathfinder save_single_paths=1"; - run_command_output out = run_command(ss.str()); - ASSERT_FALSE(out.hasError); - std::fstream single_csv_stream(convert_model_path(output_save_single)); - std::stringstream result_sstream; - result_sstream << single_csv_stream.rdbuf(); - single_csv_stream.close(); - std::string output_csv = result_sstream.str(); - EXPECT_EQ(1, count_matches("# Elapsed Time:", output_csv)); - EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", output_csv)); - EXPECT_EQ(1, count_matches("save_single_paths = 1", output_csv)); - - std::fstream single_json_stream(convert_model_path(output_save_single_diag)); - std::stringstream result_json_sstream; - result_json_sstream << single_json_stream.rdbuf(); - single_json_stream.close(); - std::string output_json = result_json_sstream.str(); - ASSERT_FALSE(output_json.empty()); - - rapidjson::Document document; - ASSERT_FALSE(document.Parse<0>(output_json.c_str()).HasParseError()); - EXPECT_EQ(1, count_matches("\"1\" : {\"iter\" : 1,", output_json)); -} - -TEST_F(CmdStan, pathfinder_single) { +TEST_F(CmdStan, pathfinder_40_draws) { std::stringstream ss; ss << convert_model_path(multi_normal_model) << " output refresh=0 file=" << convert_model_path(arg_output) - << " method=pathfinder" - << " num_paths=1"; + << " method=pathfinder num_psis_draws=40"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); @@ -110,20 +80,23 @@ TEST_F(CmdStan, pathfinder_single) { result_stream.close(); std::string output = result_sstream.str(); EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); - EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", output)); - EXPECT_EQ(1, count_matches("num_paths = 1", output)); + EXPECT_EQ(1, count_matches(" seconds (Pathfinders)", output)); + EXPECT_EQ(1, count_matches(" seconds (PSIS)", output)); + EXPECT_EQ(1, count_matches(" seconds (Total)", output)); + EXPECT_EQ(1, count_matches("num_psis_draws = 40", output)); + EXPECT_EQ(1, count_matches("num_paths = 4 (Default)", output)); EXPECT_EQ(1, count_matches("save_single_paths = 0 (Default)", output)); } -TEST_F(CmdStan, pathfinder_single_good_plus) { +TEST_F(CmdStan, pathfinder_single) { std::stringstream ss; ss << convert_model_path(multi_normal_model) << " output refresh=0 file=" << convert_model_path(arg_output) << " method=pathfinder" - << " num_paths=1" - << " save_single_paths=1"; + << " num_paths=1"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); + ASSERT_FALSE(file_exists(convert_model_path(output_json))); std::fstream result_stream(convert_model_path(output_csv)); std::stringstream result_sstream; @@ -133,30 +106,85 @@ TEST_F(CmdStan, pathfinder_single_good_plus) { EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", output)); EXPECT_EQ(1, count_matches("num_paths = 1", output)); - EXPECT_EQ(1, count_matches("save_single_paths = 1", output)); - - ASSERT_FALSE(file_exists(convert_model_path(output_save_single))); + EXPECT_EQ(1, count_matches("save_single_paths = 0 (Default)", output)); } -TEST_F(CmdStan, pathfinder_num_paths_8) { + +TEST_F(CmdStan, pathfinder_save_single_default_num_paths) { std::stringstream ss; ss << convert_model_path(multi_normal_model) << " output refresh=0 file=" << convert_model_path(arg_output) - << " method=pathfinder" - << " num_paths=8"; + << " method=pathfinder save_single_paths=1"; run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); - std::vector test_psis_path = {"test", "output.csv"}; - std::fstream result_stream(convert_model_path(test_psis_path)); + ASSERT_TRUE(file_exists(convert_model_path(output_csv))); + ASSERT_TRUE(file_exists(convert_model_path(output_single_csv))); + ASSERT_TRUE(file_exists(convert_model_path(output_single_json))); + + std::fstream single_csv_stream(convert_model_path(output_single_csv)); std::stringstream result_sstream; - result_sstream << result_stream.rdbuf(); - result_stream.close(); - std::string output = result_sstream.str(); - EXPECT_EQ(1, count_matches("num_paths = 8", output)); - EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); - EXPECT_EQ(1, count_matches(" seconds (Pathfinders)", output)); + result_sstream << single_csv_stream.rdbuf(); + single_csv_stream.close(); + std::string single_csv = result_sstream.str(); + EXPECT_EQ(1, count_matches("# Elapsed Time:", single_csv)); + EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", single_csv)); + EXPECT_EQ(1, count_matches("save_single_paths = 1", single_csv)); + + std::fstream single_json_stream(convert_model_path(output_single_json)); + std::stringstream result_json_sstream; + result_json_sstream << single_json_stream.rdbuf(); + single_json_stream.close(); + std::string single_json = result_json_sstream.str(); + ASSERT_FALSE(single_json.empty()); + + rapidjson::Document document; + ASSERT_FALSE(document.Parse<0>(single_json.c_str()).HasParseError()); + EXPECT_EQ(1, count_matches("\"1\" : {\"iter\" : 1,", single_json)); } + +// TEST_F(CmdStan, pathfinder_single_good_plus) { +// std::stringstream ss; +// ss << convert_model_path(multi_normal_model) +// << " output refresh=0 file=" << convert_model_path(arg_output) +// << " method=pathfinder" +// << " num_paths=1" +// << " save_single_paths=1"; +// run_command_output out = run_command(ss.str()); +// ASSERT_FALSE(out.hasError); + +// std::fstream result_stream(convert_model_path(output_csv)); +// std::stringstream result_sstream; +// result_sstream << result_stream.rdbuf(); +// result_stream.close(); +// std::string output = result_sstream.str(); +// EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); +// EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", output)); +// EXPECT_EQ(1, count_matches("num_paths = 1", output)); +// EXPECT_EQ(1, count_matches("save_single_paths = 1", output)); + +// ASSERT_FALSE(file_exists(convert_model_path(output_save_single))); +// } + +// TEST_F(CmdStan, pathfinder_num_paths_8) { +// std::stringstream ss; +// ss << convert_model_path(multi_normal_model) +// << " output refresh=0 file=" << convert_model_path(arg_output) +// << " method=pathfinder" +// << " num_paths=8"; +// run_command_output out = run_command(ss.str()); +// ASSERT_FALSE(out.hasError); +// std::vector test_psis_path = {"test", "output.csv"}; +// std::fstream result_stream(convert_model_path(test_psis_path)); +// std::stringstream result_sstream; +// result_sstream << result_stream.rdbuf(); +// result_stream.close(); +// std::string output = result_sstream.str(); +// EXPECT_EQ(1, count_matches("num_paths = 8", output)); +// EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); +// EXPECT_EQ(1, count_matches(" seconds (Pathfinders)", output)); +// } + // TEST_F(CmdStan, pathfinder_diagnostic_json) { // std::stringstream ss; // ss << convert_model_path(multi_normal_model) @@ -199,21 +227,21 @@ TEST_F(CmdStan, pathfinder_num_paths_8) { // EXPECT_EQ(0, count_matches("\"4\" : {\"iter\" : 4,", output)); // } -TEST_F(CmdStan, pathfinder_num_paths_draws) { - std::stringstream ss; - ss << convert_model_path(eight_schools_model) - << " data file=" << convert_model_path(eight_schools_data) - << " output refresh=0 file=" << convert_model_path(arg_output) - << " method=pathfinder num_draws=10 num_paths=2"; - run_command_output out = run_command(ss.str()); - ASSERT_FALSE(out.hasError); +// TEST_F(CmdStan, pathfinder_num_paths_draws) { +// std::stringstream ss; +// ss << convert_model_path(eight_schools_model) +// << " data file=" << convert_model_path(eight_schools_data) +// << " output refresh=0 file=" << convert_model_path(arg_output) +// << " method=pathfinder num_draws=10 num_paths=2"; +// run_command_output out = run_command(ss.str()); +// ASSERT_FALSE(out.hasError); - std::fstream result_stream(convert_model_path(output_csv)); - std::stringstream result_sstream; - result_sstream << result_stream.rdbuf(); - result_stream.close(); - std::string output = result_sstream.str(); - ASSERT_FALSE(output.empty()); - EXPECT_EQ(1, count_matches("num_paths = 2", output)); - EXPECT_EQ(1, count_matches("num_draws = 10", output)); -} +// std::fstream result_stream(convert_model_path(output_csv)); +// std::stringstream result_sstream; +// result_sstream << result_stream.rdbuf(); +// result_stream.close(); +// std::string output = result_sstream.str(); +// ASSERT_FALSE(output.empty()); +// EXPECT_EQ(1, count_matches("num_paths = 2", output)); +// EXPECT_EQ(1, count_matches("num_draws = 10", output)); +// } From e0fde95c9efeab155e58291eef219ba762a89b27 Mon Sep 17 00:00:00 2001 From: Stan Jenkins Date: Sat, 19 Aug 2023 20:06:39 -0400 Subject: [PATCH 08/13] [Jenkins] auto-formatting by clang-format version 10.0.0-4ubuntu1 --- src/cmdstan/command_helper.hpp | 55 ++++++++++++++------------ src/test/interface/pathfinder_test.cpp | 2 - 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/cmdstan/command_helper.hpp b/src/cmdstan/command_helper.hpp index f2fc182696..abb7d4b976 100644 --- a/src/cmdstan/command_helper.hpp +++ b/src/cmdstan/command_helper.hpp @@ -770,10 +770,12 @@ void init_callbacks( unsigned int num_chains = get_num_chains(parser); unsigned int id = get_arg_val(parser, "id"); int sig_figs = get_arg_val(parser, "output", "sig_figs"); - bool save_single_paths = user_method->arg("pathfinder") - && get_arg_val(parser, "method", "pathfinder", "save_single_paths"); + bool save_single_paths + = user_method->arg("pathfinder") + && get_arg_val(parser, "method", "pathfinder", + "save_single_paths"); std::string diagnostic_file - = get_arg_val(parser, "output", "diagnostic_file"); + = get_arg_val(parser, "output", "diagnostic_file"); std::vector output_filenames; std::vector diagnostic_filenames; @@ -784,37 +786,36 @@ void init_callbacks( // default - no diagnostics for (int i = 0; i < num_chains; ++i) { diag_csv_writers.emplace_back(nullptr, "# "); - diag_json_writers.emplace_back(stan::callbacks::json_writer()); + diag_json_writers.emplace_back( + stan::callbacks::json_writer()); } - + if (user_method->arg("pathfinder")) { bool inst_writers = true; bool inst_diags = true; // make filenames if (num_chains == 1) { - output_filenames - = make_filenames(get_arg_val(parser, "output", "file"), - "", ".csv", num_chains, id); + output_filenames = make_filenames( + get_arg_val(parser, "output", "file"), "", ".csv", + num_chains, id); if (!diagnostic_file.empty()) { - diagnostic_filenames - = make_filenames(get_arg_val(parser, "output", - "diagnostic_file"), - "", ".json", num_chains, id); + diagnostic_filenames = make_filenames( + get_arg_val(parser, "output", "diagnostic_file"), + "", ".json", num_chains, id); } else if (save_single_paths) { - diagnostic_filenames - = make_filenames(get_arg_val(parser, "output", - "file"), - "", ".json", num_chains, id); + diagnostic_filenames = make_filenames( + get_arg_val(parser, "output", "file"), "", ".json", + num_chains, id); } else { inst_diags = false; } } else if (save_single_paths) { // filenames for single-path outputs output_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "_path", - ".csv", num_chains, id); + get_arg_val(parser, "output", "file"), "_path", + ".csv", num_chains, id); diagnostic_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "_path", - ".json", num_chains, id); + get_arg_val(parser, "output", "file"), "_path", + ".json", num_chains, id); } else { // multi-path default: don't save single-path outputs inst_writers = false; inst_diags = false; @@ -835,18 +836,20 @@ void init_callbacks( if (inst_diags) { diag_json_writers.clear(); for (int i = 0; i < num_chains; ++i) { - auto ofs_diag = std::make_unique(diagnostic_filenames[i]); + auto ofs_diag + = std::make_unique(diagnostic_filenames[i]); if (sig_figs > -1) { ofs_diag->precision(sig_figs); } - stan::callbacks::json_writer jwriter(std::move(ofs_diag)); + stan::callbacks::json_writer jwriter( + std::move(ofs_diag)); diag_json_writers.emplace_back(std::move(jwriter)); } - } + } } else { // not pathfinder output_filenames - = make_filenames(get_arg_val(parser, "output", "file"), - "", ".csv", num_chains, id); + = make_filenames(get_arg_val(parser, "output", "file"), + "", ".csv", num_chains, id); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(output_filenames[i]); if (sig_figs > -1) @@ -856,7 +859,7 @@ void init_callbacks( if (!diagnostic_file.empty()) { diag_csv_writers.clear(); diagnostic_filenames - = make_filenames(diagnostic_file, "", ".csv", num_chains, id); + = make_filenames(diagnostic_file, "", ".csv", num_chains, id); for (int i = 0; i < num_chains; ++i) { auto ofs = std::make_unique(diagnostic_filenames[i]); if (sig_figs > -1) diff --git a/src/test/interface/pathfinder_test.cpp b/src/test/interface/pathfinder_test.cpp index 5af56bd6ef..0346036e05 100644 --- a/src/test/interface/pathfinder_test.cpp +++ b/src/test/interface/pathfinder_test.cpp @@ -109,7 +109,6 @@ TEST_F(CmdStan, pathfinder_single) { EXPECT_EQ(1, count_matches("save_single_paths = 0 (Default)", output)); } - TEST_F(CmdStan, pathfinder_save_single_default_num_paths) { std::stringstream ss; ss << convert_model_path(multi_normal_model) @@ -142,7 +141,6 @@ TEST_F(CmdStan, pathfinder_save_single_default_num_paths) { EXPECT_EQ(1, count_matches("\"1\" : {\"iter\" : 1,", single_json)); } - // TEST_F(CmdStan, pathfinder_single_good_plus) { // std::stringstream ss; // ss << convert_model_path(multi_normal_model) From 002c8f5ec49bab36ca02c364aaacd70916b7eb61 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Sun, 20 Aug 2023 14:00:58 -0400 Subject: [PATCH 09/13] updated logic and unit tests --- src/test/interface/pathfinder_test.cpp | 187 +++++++++++-------------- 1 file changed, 82 insertions(+), 105 deletions(-) diff --git a/src/test/interface/pathfinder_test.cpp b/src/test/interface/pathfinder_test.cpp index 0346036e05..ab164f55dc 100644 --- a/src/test/interface/pathfinder_test.cpp +++ b/src/test/interface/pathfinder_test.cpp @@ -19,16 +19,18 @@ class CmdStan : public testing::Test { eight_schools_data = {"src", "test", "test-models", "eight_schools.data.json"}; arg_output = {"test", "output"}; - arg_diags = {"test", "output"}; + arg_diags = {"test", "diagnostics"}; output_csv = {"test", "output.csv"}; output_json = {"test", "output.json"}; - output_single_csv = {"test", "output_path_4.csv"}; - output_single_json = {"test", "output_path_4.json"}; + output_diags = {"test", "diagnostics.json"}; + output_single_csv = {"test", "output_path_1.csv"}; + output_single_json = {"test", "output_path_1.json"}; } void TearDown() { std::remove(convert_model_path(output_csv).c_str()); std::remove(convert_model_path(output_json).c_str()); + std::remove(convert_model_path(output_diags).c_str()); std::remove(convert_model_path(output_single_csv).c_str()); std::remove(convert_model_path(output_single_json).c_str()); } @@ -41,6 +43,7 @@ class CmdStan : public testing::Test { std::vector arg_diags; std::vector output_csv; std::vector output_json; + std::vector output_diags; std::vector output_single_csv; std::vector output_single_json; }; @@ -117,6 +120,7 @@ TEST_F(CmdStan, pathfinder_save_single_default_num_paths) { run_command_output out = run_command(ss.str()); ASSERT_FALSE(out.hasError); ASSERT_TRUE(file_exists(convert_model_path(output_csv))); + ASSERT_FALSE(file_exists(convert_model_path(output_json))); ASSERT_TRUE(file_exists(convert_model_path(output_single_csv))); ASSERT_TRUE(file_exists(convert_model_path(output_single_json))); @@ -141,105 +145,78 @@ TEST_F(CmdStan, pathfinder_save_single_default_num_paths) { EXPECT_EQ(1, count_matches("\"1\" : {\"iter\" : 1,", single_json)); } -// TEST_F(CmdStan, pathfinder_single_good_plus) { -// std::stringstream ss; -// ss << convert_model_path(multi_normal_model) -// << " output refresh=0 file=" << convert_model_path(arg_output) -// << " method=pathfinder" -// << " num_paths=1" -// << " save_single_paths=1"; -// run_command_output out = run_command(ss.str()); -// ASSERT_FALSE(out.hasError); - -// std::fstream result_stream(convert_model_path(output_csv)); -// std::stringstream result_sstream; -// result_sstream << result_stream.rdbuf(); -// result_stream.close(); -// std::string output = result_sstream.str(); -// EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); -// EXPECT_EQ(1, count_matches(" seconds (Pathfinder)", output)); -// EXPECT_EQ(1, count_matches("num_paths = 1", output)); -// EXPECT_EQ(1, count_matches("save_single_paths = 1", output)); - -// ASSERT_FALSE(file_exists(convert_model_path(output_save_single))); -// } - -// TEST_F(CmdStan, pathfinder_num_paths_8) { -// std::stringstream ss; -// ss << convert_model_path(multi_normal_model) -// << " output refresh=0 file=" << convert_model_path(arg_output) -// << " method=pathfinder" -// << " num_paths=8"; -// run_command_output out = run_command(ss.str()); -// ASSERT_FALSE(out.hasError); -// std::vector test_psis_path = {"test", "output.csv"}; -// std::fstream result_stream(convert_model_path(test_psis_path)); -// std::stringstream result_sstream; -// result_sstream << result_stream.rdbuf(); -// result_stream.close(); -// std::string output = result_sstream.str(); -// EXPECT_EQ(1, count_matches("num_paths = 8", output)); -// EXPECT_EQ(1, count_matches("# Elapsed Time:", output)); -// EXPECT_EQ(1, count_matches(" seconds (Pathfinders)", output)); -// } - -// TEST_F(CmdStan, pathfinder_diagnostic_json) { -// std::stringstream ss; -// ss << convert_model_path(multi_normal_model) -// << " output refresh=0 file=" << convert_model_path(arg_output) -// << " diagnostic_file=" << convert_model_path(arg_diags) -// << " method=pathfinder"; -// run_command_output out = run_command(ss.str()); -// ASSERT_FALSE(out.hasError); - -// std::fstream result_stream(convert_model_path(output_diags)); -// std::stringstream result_sstream; -// result_sstream << result_stream.rdbuf(); -// result_stream.close(); -// std::string output = result_sstream.str(); -// ASSERT_FALSE(output.empty()); -// rapidjson::Document document; -// ASSERT_FALSE(document.Parse<0>(output.c_str()).HasParseError()); -// } - -// TEST_F(CmdStan, pathfinder_lbfgs_iterations) { -// std::stringstream ss; -// ss << convert_model_path(eight_schools_model) -// << " data file=" << convert_model_path(eight_schools_data) -// << " random seed=12345" -// << " output refresh=0 file=" << convert_model_path(arg_output) -// << " diagnostic_file=" << convert_model_path(arg_diags) -// << " method=pathfinder max_lbfgs_iters=3"; -// run_command_output out = run_command(ss.str()); -// ASSERT_FALSE(out.hasError); - -// std::fstream result_stream(convert_model_path(output_diags)); -// std::stringstream result_sstream; -// result_sstream << result_stream.rdbuf(); -// result_stream.close(); -// std::string output = result_sstream.str(); -// ASSERT_FALSE(output.empty()); -// rapidjson::Document document; -// ASSERT_FALSE(document.Parse<0>(output.c_str()).HasParseError()); -// EXPECT_EQ(1, count_matches("\"3\" : {\"iter\" : 3,", output)); -// EXPECT_EQ(0, count_matches("\"4\" : {\"iter\" : 4,", output)); -// } - -// TEST_F(CmdStan, pathfinder_num_paths_draws) { -// std::stringstream ss; -// ss << convert_model_path(eight_schools_model) -// << " data file=" << convert_model_path(eight_schools_data) -// << " output refresh=0 file=" << convert_model_path(arg_output) -// << " method=pathfinder num_draws=10 num_paths=2"; -// run_command_output out = run_command(ss.str()); -// ASSERT_FALSE(out.hasError); - -// std::fstream result_stream(convert_model_path(output_csv)); -// std::stringstream result_sstream; -// result_sstream << result_stream.rdbuf(); -// result_stream.close(); -// std::string output = result_sstream.str(); -// ASSERT_FALSE(output.empty()); -// EXPECT_EQ(1, count_matches("num_paths = 2", output)); -// EXPECT_EQ(1, count_matches("num_draws = 10", output)); -// } +TEST_F(CmdStan, pathfinder_save_single_num_paths_1) { + std::stringstream ss; + ss << convert_model_path(multi_normal_model) + << " output refresh=0 file=" << convert_model_path(arg_output) + << " method=pathfinder" + << " num_paths=1 save_single_paths=1"; + run_command_output out = run_command(ss.str()); + ASSERT_FALSE(out.hasError); + ASSERT_TRUE(file_exists(convert_model_path(output_csv))); + ASSERT_TRUE(file_exists(convert_model_path(output_json))); + ASSERT_FALSE(file_exists(convert_model_path(output_single_csv))); + ASSERT_FALSE(file_exists(convert_model_path(output_single_json))); +} + +TEST_F(CmdStan, pathfinder_save_single_num_paths_1_diag_file_arg) { + std::stringstream ss; + ss << convert_model_path(multi_normal_model) + << " output refresh=0 file=" << convert_model_path(arg_output) + << " diagnostic_file=" << convert_model_path(arg_diags) + << " method=pathfinder" + << " num_paths=1 save_single_paths=1"; + run_command_output out = run_command(ss.str()); + ASSERT_FALSE(out.hasError); + ASSERT_TRUE(file_exists(convert_model_path(output_csv))); + ASSERT_TRUE(file_exists(convert_model_path(output_diags))); + ASSERT_FALSE(file_exists(convert_model_path(output_json))); + ASSERT_FALSE(file_exists(convert_model_path(output_single_csv))); + ASSERT_FALSE(file_exists(convert_model_path(output_single_json))); +} + +TEST_F(CmdStan, pathfinder_num_paths_8) { + std::stringstream ss; + ss << convert_model_path(multi_normal_model) + << " output refresh=0 file=" << convert_model_path(arg_output) + << " method=pathfinder" + << " num_paths=8"; + run_command_output out = run_command(ss.str()); + ASSERT_FALSE(out.hasError); + ASSERT_TRUE(file_exists(convert_model_path(output_csv))); + ASSERT_FALSE(file_exists(convert_model_path(output_single_csv))); + + std::fstream result_stream(convert_model_path(output_csv)); + std::stringstream result_sstream; + result_sstream << result_stream.rdbuf(); + result_stream.close(); + std::string output = result_sstream.str(); + EXPECT_EQ(1, count_matches(" seconds (Pathfinders)", output)); + EXPECT_EQ(1, count_matches(" seconds (PSIS)", output)); + EXPECT_EQ(1, count_matches("num_paths = 8", output)); +} + +TEST_F(CmdStan, pathfinder_lbfgs_iterations) { + std::stringstream ss; + ss << convert_model_path(eight_schools_model) + << " data file=" << convert_model_path(eight_schools_data) + << " random seed=12345" + << " output refresh=0 file=" << convert_model_path(arg_output) + << " method=pathfinder max_lbfgs_iters=3" + << " save_single_paths=1"; + run_command_output out = run_command(ss.str()); + ASSERT_FALSE(out.hasError); + ASSERT_TRUE(file_exists(convert_model_path(output_csv))); + ASSERT_TRUE(file_exists(convert_model_path(output_single_json))); + + std::fstream result_stream(convert_model_path(output_single_json)); + std::stringstream result_sstream; + result_sstream << result_stream.rdbuf(); + result_stream.close(); + std::string output = result_sstream.str(); + ASSERT_FALSE(output.empty()); + rapidjson::Document document; + ASSERT_FALSE(document.Parse<0>(output.c_str()).HasParseError()); + EXPECT_EQ(1, count_matches("\"3\" : {\"iter\" : 3,", output)); + EXPECT_EQ(0, count_matches("\"4\" : {\"iter\" : 4,", output)); +} From 309549662fbb224496f33c74cca58fa47eb45efb Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Sun, 20 Aug 2023 17:39:36 -0400 Subject: [PATCH 10/13] tweak pathfinder args ordering --- src/cmdstan/arguments/arg_pathfinder.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmdstan/arguments/arg_pathfinder.hpp b/src/cmdstan/arguments/arg_pathfinder.hpp index b5b0671edb..1999807462 100644 --- a/src/cmdstan/arguments/arg_pathfinder.hpp +++ b/src/cmdstan/arguments/arg_pathfinder.hpp @@ -16,11 +16,11 @@ class arg_pathfinder : public arg_lbfgs { _subarguments.push_back(new arg_single_int_pos( "num_psis_draws", "Number of draws from PSIS sample", 1000)); + _subarguments.push_back( + new arg_single_int_pos("num_paths", "Number of single pathfinders", 4)); _subarguments.push_back(new arg_single_bool( "save_single_paths", "Output single-path pathfinder draws as CSV", false)); - _subarguments.push_back( - new arg_single_int_pos("num_paths", "Number of single pathfinders", 4)); _subarguments.push_back(new arg_single_int_pos( "max_lbfgs_iters", "Maximum number of LBFGS iterations", 1000)); _subarguments.push_back(new arg_single_int_pos( From b4be8cccd3ad68dc2f95f825333fdde7502e4527 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Mon, 21 Aug 2023 13:45:29 -0400 Subject: [PATCH 11/13] changes per code review --- src/cmdstan/command_helper.hpp | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/cmdstan/command_helper.hpp b/src/cmdstan/command_helper.hpp index abb7d4b976..3077db5de5 100644 --- a/src/cmdstan/command_helper.hpp +++ b/src/cmdstan/command_helper.hpp @@ -744,7 +744,9 @@ std::vector make_filenames(const std::string &filename, unsigned int id) { std::vector names(num_chains); auto base_sfx = get_basename_suffix(filename); - base_sfx.second = type; + if (base_sfx.second.empty()) { + base_sfx.second = type; + } auto name_iterator = [num_chains, id](auto i) { if (num_chains == 1) { return std::string(""); @@ -774,11 +776,12 @@ void init_callbacks( = user_method->arg("pathfinder") && get_arg_val(parser, "method", "pathfinder", "save_single_paths"); + std::string output_file + = get_arg_val(parser, "output", "file"); std::string diagnostic_file = get_arg_val(parser, "output", "diagnostic_file"); std::vector output_filenames; std::vector diagnostic_filenames; - sample_writers.reserve(num_chains); diag_csv_writers.reserve(num_chains); diag_json_writers.reserve(num_chains); @@ -791,31 +794,24 @@ void init_callbacks( } if (user_method->arg("pathfinder")) { + std::string basename = get_basename_suffix(output_file).first; + std::string diag_basename = get_basename_suffix(diagnostic_file).first; bool inst_writers = true; bool inst_diags = true; - // make filenames if (num_chains == 1) { - output_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "", ".csv", - num_chains, id); - if (!diagnostic_file.empty()) { - diagnostic_filenames = make_filenames( - get_arg_val(parser, "output", "diagnostic_file"), - "", ".json", num_chains, id); + output_filenames.emplace_back(basename + ".csv"); + if (!diag_basename.empty()) { + diagnostic_filenames.emplace_back(diag_basename + ".json"); } else if (save_single_paths) { - diagnostic_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "", ".json", - num_chains, id); + diagnostic_filenames.emplace_back(basename + ".json"); } else { inst_diags = false; } } else if (save_single_paths) { // filenames for single-path outputs - output_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "_path", - ".csv", num_chains, id); - diagnostic_filenames = make_filenames( - get_arg_val(parser, "output", "file"), "_path", - ".json", num_chains, id); + output_filenames + = make_filenames(basename, "_path", ".csv", num_chains, id); + diagnostic_filenames + = make_filenames(basename, "_path", ".json", num_chains, id); } else { // multi-path default: don't save single-path outputs inst_writers = false; inst_diags = false; From 71e54dc8961fe8d69cd22fbdd7641b0f0e026a4c Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Mon, 21 Aug 2023 14:07:50 -0400 Subject: [PATCH 12/13] changes per code review --- src/cmdstan/command.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/cmdstan/command.hpp b/src/cmdstan/command.hpp index 5bbeedd948..b27454092a 100644 --- a/src/cmdstan/command.hpp +++ b/src/cmdstan/command.hpp @@ -267,10 +267,11 @@ int command(int argc, const char *argv[]) { save_iterations, refresh, interrupt, logger, init_writer, sample_writers[0], diagnostic_json_writers[0]); } else { - std::vector pf_names = make_filenames( - get_arg_val(parser, "output", "file"), "", ".csv", 1, - id); - auto ofs = std::make_unique(pf_names[0]); + auto ofs + = std::make_unique( + get_basename_suffix( + get_arg_val(parser, "output", "file")).first + + ".csv"); if (sig_figs > -1) ofs->precision(sig_figs); stan::callbacks::unique_stream_writer pathfinder_writer( From 4e647f00877ecd570617d04685caa2ddb56af042 Mon Sep 17 00:00:00 2001 From: Stan Jenkins Date: Mon, 21 Aug 2023 14:12:36 -0400 Subject: [PATCH 13/13] [Jenkins] auto-formatting by clang-format version 10.0.0-4ubuntu1 --- src/cmdstan/command.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cmdstan/command.hpp b/src/cmdstan/command.hpp index b27454092a..0addc5021c 100644 --- a/src/cmdstan/command.hpp +++ b/src/cmdstan/command.hpp @@ -267,11 +267,11 @@ int command(int argc, const char *argv[]) { save_iterations, refresh, interrupt, logger, init_writer, sample_writers[0], diagnostic_json_writers[0]); } else { - auto ofs - = std::make_unique( - get_basename_suffix( - get_arg_val(parser, "output", "file")).first - + ".csv"); + auto ofs = std::make_unique( + get_basename_suffix( + get_arg_val(parser, "output", "file")) + .first + + ".csv"); if (sig_figs > -1) ofs->precision(sig_figs); stan::callbacks::unique_stream_writer pathfinder_writer(