Issue/3200 pathfinder diagnostic

src/stan/callbacks/json_writer.hpp

@@ -38,14 +38,17 @@ class json_writer {
   // separator)
   int record_depth_ = 0;
   // Whether or not the record's parent object needs a comma separator
-  bool record_needs_comma_;
+  bool record_needs_comma_ = false;
 
   /**
    * Writes a comma separator for the record's parent object if needed.
    */
   void write_record_comma_if_needed() {
     if (record_depth_ > 0 && record_needs_comma_) {
-      *output_ << ",";
+      *output_ << ",\n";
+      record_needs_comma_ = false;
+    } else {
+      write_sep();
     }
   }
 
@@ -265,8 +268,8 @@ class json_writer {
       return;
     write_record_comma_if_needed();
     *output_ << "{";
-    record_needs_comma_ = false;
     record_depth_++;
+    record_element_needs_comma_ = false;
   }
 
   /**
@@ -277,9 +280,9 @@ class json_writer {
     if (output_ == nullptr)
       return;
     write_record_comma_if_needed();
-    *output_ << "\"" << key << "\": {";
-    record_needs_comma_ = false;
+    *output_ << "\"" << key << "\" : {";
     record_depth_++;
+    record_element_needs_comma_ = false;
   }
   /**
    * Writes "}", final token of a JSON record.
@@ -291,8 +294,9 @@ class json_writer {
     record_depth_--;
     if (record_depth_ > 0) {
       record_needs_comma_ = true;
+    } else {
+      *output_ << "\n";
     }
-    record_element_needs_comma_ = false;
   }
 
   /**

src/stan/callbacks/structured_writer.hpp

@@ -19,25 +19,30 @@ class structured_writer {
    * Virtual destructor.
    */
   virtual ~structured_writer() {}
+
   /**
-   * Writes "{", initial token of a JSON record.
+   * Writes start token of a structured record.
    */
   virtual void begin_record() {}
+
   /**
-   * Writes "\"key\" : {", initial token of a named JSON record.
+   * Writes key followed by start token of a structured record.
    * @param[in] key The name of the record.
    */
-  virtual void begin_record(const std::string&) {}
+  virtual void begin_record(const std::string&, bool newline = false) {}
+
   /**
-   * Writes "}", final token of a JSON record.
+   * Writes end token of a structured record.
    */
   virtual void end_record() {}
+
   /**
-   * Writes "[", initial token of a JSON list.
+   * Writes start token of a list.
    */
   virtual void begin_list() {}
+
   /**
-   * Writes "]", final token of a JSON list.
+   * Writes end token of a list.
    */
   virtual void end_list() {}
 
@@ -47,16 +52,19 @@ class structured_writer {
    * @param key Name of the value pair
    */
   virtual void write(const std::string& key) {}
+
   /**
    * Write a key-value pair where the value is a string.
    * @param key Name of the value pair
    * @param value string to write.
    */
   virtual void write(const std::string& key, const std::string& value) {}
+
   /**
    * No-op
    */
   virtual void write() {}
+
   /**
    * Write a key-value pair where the value is a bool.
    * @param key Name of the value pair
@@ -70,18 +78,21 @@ class structured_writer {
    * @param value int to write.
    */
   virtual void write(const std::string& key, int value) {}
+
   /**
    * Write a key-value pair where the value is an `std::size_t`.
    * @param key Name of the value pair
    * @param value `std::size_t` to write.
    */
   virtual void write(const std::string& key, std::size_t value) {}
+
   /**
    * Write a key-value pair where the value is a double.
    * @param key Name of the value pair
    * @param value double to write.
    */
   virtual void write(const std::string& key, double value) {}
+
   /**
    * Write a key-value pair where the value is a complex value.
    * @param key Name of the value pair
@@ -96,25 +107,29 @@ class structured_writer {
    */
   virtual void write(const std::string& key, const std::vector<double> values) {
   }
+
   /**
    * Write a key-value pair where the value is a vector of strings to be made a
    * list.
    * @param key Name of the value pair
    * @param values vector of strings to write.
    */
   void write(const std::string& key, const std::vector<std::string>& values) {}
+
   /**
    * Write a key-value pair where the value is an Eigen Matrix.
    * @param key Name of the value pair
    * @param mat Eigen Matrix to write.
    */
   void write(const std::string& key, const Eigen::MatrixXd& mat) {}
+
   /**
    * Write a key-value pair where the value is an Eigen Vector.
    * @param key Name of the value pair
    * @param vec Eigen Vector to write.
    */
   void write(const std::string& key, const Eigen::VectorXd& vec) {}
+
   /**
    * Write a key-value pair where the value is a Eigen RowVector.
    * @param key Name of the value pair

src/stan/callbacks/writer.hpp

@@ -2,7 +2,6 @@
 #define STAN_CALLBACKS_WRITER_HPP
 
 #include <stan/math/prim/fun/Eigen.hpp>
-#include <boost/lexical_cast.hpp>
 #include <string>
 #include <vector>
 

src/stan/io/dump.hpp

@@ -5,7 +5,6 @@
 #include <stan/io/validate_dims.hpp>
 #include <stan/io/var_context.hpp>
 #include <stan/math/prim.hpp>
-#include <boost/lexical_cast.hpp>
 #include <iostream>
 #include <limits>
 #include <map>
@@ -242,8 +241,8 @@ class dump_reader {
     scan_optional_long();
     size_t d = 0;
     try {
-      d = boost::lexical_cast<size_t>(buf_);
-    } catch (const boost::bad_lexical_cast& exc) {
+      d = std::stoull(buf_);
+    } catch (const std::exception& exc) {
       std::string msg = "value " + buf_ + " beyond array dimension range";
       throw std::invalid_argument(msg);
     }
@@ -269,8 +268,8 @@ class dump_reader {
   int get_int() {
     int n = 0;
     try {
-      n = boost::lexical_cast<int>(buf_);
-    } catch (const boost::bad_lexical_cast& exc) {
+      n = std::stol(buf_);
+    } catch (const std::exception& exc) {
       std::string msg = "value " + buf_ + " beyond int range";
       throw std::invalid_argument(msg);
     }
@@ -280,17 +279,17 @@ class dump_reader {
   double scan_double() {
     double x = 0;
     try {
-      x = boost::lexical_cast<double>(buf_);
+      x = std::stod(buf_);
       if (x == 0)
         validate_zero_buf(buf_);
-    } catch (const boost::bad_lexical_cast& exc) {
+    } catch (const std::exception& exc) {
       std::string msg = "value " + buf_ + " beyond numeric range";
       throw std::invalid_argument(msg);
     }
     return x;
   }
 
-  // scan number stores number or throws bad lexical cast exception
+  // scan number stores number or throws exception
   void scan_number(bool negate_val) {
     // must take longest first!
     if (scan_chars("Inf")) {

src/stan/io/validate_zero_buf.hpp

@@ -14,7 +14,7 @@ namespace io {
  * operator[](size_t)</code>.
  *
  * @tparam B Character buffer type
- * @throw <code>boost::bad_lexical_cast</code> if the buffer
+ * @throw <code>std::bad_cast</code> if the buffer
  * contains non-zero characters before an exponentiation symbol.
  */
 template <typename B>
@@ -23,7 +23,7 @@ void validate_zero_buf(const B& buf) {
     if (buf[i] == 'e' || buf[i] == 'E')
       return;
     if (buf[i] >= '1' && buf[i] <= '9')
-      boost::conversion::detail::throw_bad_cast<std::string, double>();
+      throw std::bad_cast();
   }
 }
 

src/stan/mcmc/hmc/hamiltonians/ps_point.hpp

@@ -3,7 +3,6 @@
 
 #include <stan/callbacks/writer.hpp>
 #include <stan/math/prim/fun/Eigen.hpp>
-#include <boost/lexical_cast.hpp>
 #include <string>
 #include <vector>
 

src/stan/variational/advi.hpp

@@ -11,7 +11,6 @@
 #include <stan/variational/families/normal_fullrank.hpp>
 #include <stan/variational/families/normal_meanfield.hpp>
 #include <boost/circular_buffer.hpp>
-#include <boost/lexical_cast.hpp>
 #include <algorithm>
 #include <chrono>
 #include <limits>

src/test/unit/callbacks/json_writer_test.cpp

@@ -22,10 +22,30 @@ class StanInterfaceCallbacksJsonWriter : public ::testing::Test {
   stan::callbacks::json_writer<std::stringstream, deleter_noop> writer;
 };
 
-TEST_F(StanInterfaceCallbacksJsonWriter, begin_record_end) {
+TEST_F(StanInterfaceCallbacksJsonWriter, begin_end_record) {
   writer.begin_record();
   writer.end_record();
-  EXPECT_EQ("{}", ss.str());
+  EXPECT_EQ("{}\n", ss.str());
+}
+
+TEST_F(StanInterfaceCallbacksJsonWriter, begin_end_record_nested) {
+  std::string key("key");
+  std::string value("value");
+  writer.begin_record();
+  writer.begin_record("1");
+  writer.write(key, value);
+  writer.end_record();
+  writer.begin_record("2");
+  writer.write(key, value);
+  writer.begin_record("2.1");
+  writer.write(key, value);
+  writer.end_record();
+  writer.end_record();
+  writer.end_record();
+  EXPECT_EQ(
+      "{\"1\" : {\"key\" : \"value\"},\n\"2\" : {\"key\" : \"value\","
+      " \"2.1\" : {\"key\" : \"value\"}}}\n",
+      ss.str());
 }
 
 TEST_F(StanInterfaceCallbacksJsonWriter, write_double_vector) {
@@ -58,7 +78,7 @@ TEST_F(StanInterfaceCallbacksJsonWriter, single_member) {
   writer.begin_record();
   writer.write(key, value);
   writer.end_record();
-  EXPECT_EQ("{\"key\" : \"value\"}", ss.str());
+  EXPECT_EQ("{\"key\" : \"value\"}\n", ss.str());
 }
 
 TEST_F(StanInterfaceCallbacksJsonWriter, more_members) {
@@ -68,18 +88,12 @@ TEST_F(StanInterfaceCallbacksJsonWriter, more_members) {
   writer.write(key, value);
   writer.write(key, value);
 
-  EXPECT_EQ(
-      "{\"key\" : \"value\""
-      ", \"key\" : \"value\"",
-      ss.str());
+  EXPECT_EQ("{\"key\" : \"value\", \"key\" : \"value\"", ss.str());
 
   writer.write(key, value);
   writer.end_record();
-  EXPECT_EQ(
-      "{\"key\" : \"value\""
-      ", \"key\" : \"value\""
-      ", \"key\" : \"value\"}",
-      ss.str());
+  EXPECT_EQ("{\"key\" : \"value\", \"key\" : \"value\", \"key\" : \"value\"}\n",
+            ss.str());
 }
 
 TEST_F(StanInterfaceCallbacksJsonWriter, write_double_vector_precision2) {

src/test/unit/io/validate_zero_buf_test.cpp

@@ -15,8 +15,8 @@ TEST(ioValidateZeroBuf, tester) {
   EXPECT_NO_THROW(validate_zero_buf(s));
 
   s = "1.0";
-  EXPECT_THROW(validate_zero_buf(s), boost::bad_lexical_cast);
+  EXPECT_THROW(validate_zero_buf(s), std::bad_cast);
 
   s = "1e1";
-  EXPECT_THROW(validate_zero_buf(s), boost::bad_lexical_cast);
+  EXPECT_THROW(validate_zero_buf(s), std::bad_cast);
 }

src/test/unit/services/pathfinder/eight_schools_test.cpp

@@ -1,6 +1,6 @@
 #include <stan/callbacks/json_writer.hpp>
 #include <stan/callbacks/stream_writer.hpp>
-#include <stan/callbacks/structured_writer.hpp>
+#include <stan/callbacks/json_writer.hpp>
 #include <stan/math.hpp>
 #include <stan/io/array_var_context.hpp>
 #include <stan/io/empty_var_context.hpp>
@@ -82,8 +82,8 @@ TEST_F(ServicesPathfinderEightSchools, multi) {
   std::unique_ptr<std::ostream> empty_ostream(nullptr);
   stan::test::test_logger logger(std::move(empty_ostream));
   std::vector<stan::callbacks::writer> single_path_parameter_writer(num_paths);
-  std::vector<stan::callbacks::structured_writer> single_path_diagnostic_writer(
-      num_paths);
+  std::vector<stan::callbacks::json_writer<std::stringstream>>
+      single_path_diagnostic_writer(num_paths);
   std::vector<std::unique_ptr<decltype(init_init_context())>> single_path_inits;
   for (int i = 0; i < num_paths; ++i) {
     single_path_inits.emplace_back(

src/test/unit/services/pathfinder/normal_glm_test.cpp

@@ -10,6 +10,7 @@
 #include <test/unit/services/instrumented_callbacks.hpp>
 #include <test/unit/services/pathfinder/util.hpp>
 #include <test/unit/services/util.hpp>
+#include <rapidjson/document.h>
 #include <gtest/gtest.h>
 
 // Locally tests can use threads but for jenkins we should just use 1 thread
@@ -131,6 +132,8 @@ TEST_F(ServicesPathfinderGLM, single) {
   for (int i = 2; i < all_mean_vals.cols(); ++i) {
     EXPECT_NEAR(0, all_sd_vals(2, i), .1);
   }
+  rapidjson::Document document;
+  ASSERT_FALSE(document.Parse<0>(diagnostic_ss.str().c_str()).HasParseError());
 }
 
 TEST_F(ServicesPathfinderGLM, multi) {