diff --git a/src/stan/callbacks/json_writer.hpp b/src/stan/callbacks/json_writer.hpp
index 7f8d254e67..54f54bf385 100644
--- a/src/stan/callbacks/json_writer.hpp
+++ b/src/stan/callbacks/json_writer.hpp
@@ -38,14 +38,17 @@ class json_writer {
   // separator)
   int record_depth_ = 0;
   // Whether or not the record's parent object needs a comma separator
-  bool record_needs_comma_;
+  bool record_needs_comma_ = false;
 
   /**
    * Writes a comma separator for the record's parent object if needed.
    */
   void write_record_comma_if_needed() {
     if (record_depth_ > 0 && record_needs_comma_) {
-      *output_ << ",";
+      *output_ << ",\n";
+      record_needs_comma_ = false;
+    } else {
+      write_sep();
     }
   }
 
@@ -265,8 +268,8 @@ class json_writer {
       return;
     write_record_comma_if_needed();
     *output_ << "{";
-    record_needs_comma_ = false;
     record_depth_++;
+    record_element_needs_comma_ = false;
   }
 
   /**
@@ -277,9 +280,9 @@ class json_writer {
     if (output_ == nullptr)
       return;
     write_record_comma_if_needed();
-    *output_ << "\"" << key << "\": {";
-    record_needs_comma_ = false;
+    *output_ << "\"" << key << "\" : {";
     record_depth_++;
+    record_element_needs_comma_ = false;
   }
   /**
    * Writes "}", final token of a JSON record.
@@ -291,8 +294,9 @@ class json_writer {
     record_depth_--;
     if (record_depth_ > 0) {
       record_needs_comma_ = true;
+    } else {
+      *output_ << "\n";
     }
-    record_element_needs_comma_ = false;
   }
 
   /**
diff --git a/src/stan/callbacks/structured_writer.hpp b/src/stan/callbacks/structured_writer.hpp
index 3b03f8a80a..1d85400591 100644
--- a/src/stan/callbacks/structured_writer.hpp
+++ b/src/stan/callbacks/structured_writer.hpp
@@ -19,25 +19,30 @@ class structured_writer {
    * Virtual destructor.
    */
   virtual ~structured_writer() {}
+
   /**
-   * Writes "{", initial token of a JSON record.
+   * Writes start token of a structured record.
    */
   virtual void begin_record() {}
+
   /**
-   * Writes "\"key\" : {", initial token of a named JSON record.
+   * Writes key followed by start token of a structured record.
    * @param[in] key The name of the record.
    */
-  virtual void begin_record(const std::string&) {}
+  virtual void begin_record(const std::string&, bool newline = false) {}
+
   /**
-   * Writes "}", final token of a JSON record.
+   * Writes end token of a structured record.
    */
   virtual void end_record() {}
+
   /**
-   * Writes "[", initial token of a JSON list.
+   * Writes start token of a list.
    */
   virtual void begin_list() {}
+
   /**
-   * Writes "]", final token of a JSON list.
+   * Writes end token of a list.
    */
   virtual void end_list() {}
 
@@ -47,16 +52,19 @@ class structured_writer {
    * @param key Name of the value pair
    */
   virtual void write(const std::string& key) {}
+
   /**
    * Write a key-value pair where the value is a string.
    * @param key Name of the value pair
    * @param value string to write.
    */
   virtual void write(const std::string& key, const std::string& value) {}
+
   /**
    * No-op
    */
   virtual void write() {}
+
   /**
    * Write a key-value pair where the value is a bool.
    * @param key Name of the value pair
@@ -70,18 +78,21 @@ class structured_writer {
    * @param value int to write.
    */
   virtual void write(const std::string& key, int value) {}
+
   /**
    * Write a key-value pair where the value is an `std::size_t`.
    * @param key Name of the value pair
    * @param value `std::size_t` to write.
    */
   virtual void write(const std::string& key, std::size_t value) {}
+
   /**
    * Write a key-value pair where the value is a double.
    * @param key Name of the value pair
    * @param value double to write.
    */
   virtual void write(const std::string& key, double value) {}
+
   /**
    * Write a key-value pair where the value is a complex value.
    * @param key Name of the value pair
@@ -96,6 +107,7 @@ class structured_writer {
    */
   virtual void write(const std::string& key, const std::vector<double> values) {
   }
+
   /**
    * Write a key-value pair where the value is a vector of strings to be made a
    * list.
@@ -103,18 +115,21 @@ class structured_writer {
    * @param values vector of strings to write.
    */
   void write(const std::string& key, const std::vector<std::string>& values) {}
+
   /**
    * Write a key-value pair where the value is an Eigen Matrix.
    * @param key Name of the value pair
    * @param mat Eigen Matrix to write.
    */
   void write(const std::string& key, const Eigen::MatrixXd& mat) {}
+
   /**
    * Write a key-value pair where the value is an Eigen Vector.
    * @param key Name of the value pair
    * @param vec Eigen Vector to write.
    */
   void write(const std::string& key, const Eigen::VectorXd& vec) {}
+
   /**
    * Write a key-value pair where the value is a Eigen RowVector.
    * @param key Name of the value pair
diff --git a/src/test/unit/callbacks/json_writer_test.cpp b/src/test/unit/callbacks/json_writer_test.cpp
index 2d2ec831d2..71893db858 100644
--- a/src/test/unit/callbacks/json_writer_test.cpp
+++ b/src/test/unit/callbacks/json_writer_test.cpp
@@ -1,6 +1,7 @@
-#include <gtest/gtest.h>
-#include <boost/lexical_cast.hpp>
 #include <stan/callbacks/json_writer.hpp>
+#include <rapidjson/document.h>
+#include <gtest/gtest.h>
+#include <string>
 
 struct deleter_noop {
   template <typename T>
@@ -22,10 +23,39 @@ class StanInterfaceCallbacksJsonWriter : public ::testing::Test {
   stan::callbacks::json_writer<std::stringstream, deleter_noop> writer;
 };
 
-TEST_F(StanInterfaceCallbacksJsonWriter, begin_record_end) {
+TEST_F(StanInterfaceCallbacksJsonWriter, begin_end_record) {
+  writer.begin_record();
+  writer.end_record();
+  EXPECT_EQ("{}\n", ss.str());
+}
+
+TEST_F(StanInterfaceCallbacksJsonWriter, begin_end_record_nested) {
+  std::string key("key");
+  std::string value("value");
   writer.begin_record();
+  writer.begin_record("1");
+  writer.write(key, value);
+  writer.end_record();  // 1
+  writer.begin_record("2");
+  writer.write(key, value);
+  writer.begin_record("2.1");
+  writer.write(key, value);
+  writer.write(key, value);
+  writer.end_record();  // 2.1
+  writer.begin_record("2.2");
+  writer.write(key, value);
+  writer.write(key, value);
+  writer.end_record();  // 2.2
+  writer.end_record();  // 2
   writer.end_record();
-  EXPECT_EQ("{}", ss.str());
+  EXPECT_EQ(
+      "{\"1\" : {\"key\" : \"value\"},\n"
+      "\"2\" : {\"key\" : \"value\","
+      " \"2.1\" : {\"key\" : \"value\", \"key\" : \"value\"},\n"
+      "\"2.2\" : {\"key\" : \"value\", \"key\" : \"value\"}}}\n",
+      ss.str());
+  rapidjson::Document document;
+  ASSERT_FALSE(document.Parse<0>(ss.str().c_str()).HasParseError());
 }
 
 TEST_F(StanInterfaceCallbacksJsonWriter, write_double_vector) {
@@ -58,7 +88,7 @@ TEST_F(StanInterfaceCallbacksJsonWriter, single_member) {
   writer.begin_record();
   writer.write(key, value);
   writer.end_record();
-  EXPECT_EQ("{\"key\" : \"value\"}", ss.str());
+  EXPECT_EQ("{\"key\" : \"value\"}\n", ss.str());
 }
 
 TEST_F(StanInterfaceCallbacksJsonWriter, more_members) {
@@ -68,18 +98,12 @@ TEST_F(StanInterfaceCallbacksJsonWriter, more_members) {
   writer.write(key, value);
   writer.write(key, value);
 
-  EXPECT_EQ(
-      "{\"key\" : \"value\""
-      ", \"key\" : \"value\"",
-      ss.str());
+  EXPECT_EQ("{\"key\" : \"value\", \"key\" : \"value\"", ss.str());
 
   writer.write(key, value);
   writer.end_record();
-  EXPECT_EQ(
-      "{\"key\" : \"value\""
-      ", \"key\" : \"value\""
-      ", \"key\" : \"value\"}",
-      ss.str());
+  EXPECT_EQ("{\"key\" : \"value\", \"key\" : \"value\", \"key\" : \"value\"}\n",
+            ss.str());
 }
 
 TEST_F(StanInterfaceCallbacksJsonWriter, write_double_vector_precision2) {
@@ -125,7 +149,7 @@ TEST_F(StanInterfaceCallbacksJsonWriter, write_string_vector) {
   const int N = 5;
   std::vector<std::string> x;
   for (int n = 0; n < N; ++n)
-    x.push_back(boost::lexical_cast<std::string>(n));
+    x.push_back(std::to_string(n));
 
   writer.write("key", x);
   EXPECT_EQ("\"key\" : [ 0, 1, 2, 3, 4 ]", ss.str());
diff --git a/src/test/unit/services/pathfinder/eight_schools_test.cpp b/src/test/unit/services/pathfinder/eight_schools_test.cpp
index 9f7749d938..8955f5e9e4 100644
--- a/src/test/unit/services/pathfinder/eight_schools_test.cpp
+++ b/src/test/unit/services/pathfinder/eight_schools_test.cpp
@@ -1,6 +1,6 @@
 #include <stan/callbacks/json_writer.hpp>
 #include <stan/callbacks/stream_writer.hpp>
-#include <stan/callbacks/structured_writer.hpp>
+#include <stan/callbacks/json_writer.hpp>
 #include <stan/math.hpp>
 #include <stan/io/array_var_context.hpp>
 #include <stan/io/empty_var_context.hpp>
@@ -82,8 +82,8 @@ TEST_F(ServicesPathfinderEightSchools, multi) {
   std::unique_ptr<std::ostream> empty_ostream(nullptr);
   stan::test::test_logger logger(std::move(empty_ostream));
   std::vector<stan::callbacks::writer> single_path_parameter_writer(num_paths);
-  std::vector<stan::callbacks::structured_writer> single_path_diagnostic_writer(
-      num_paths);
+  std::vector<stan::callbacks::json_writer<std::stringstream>>
+      single_path_diagnostic_writer(num_paths);
   std::vector<std::unique_ptr<decltype(init_init_context())>> single_path_inits;
   for (int i = 0; i < num_paths; ++i) {
     single_path_inits.emplace_back(
diff --git a/src/test/unit/services/pathfinder/normal_glm_test.cpp b/src/test/unit/services/pathfinder/normal_glm_test.cpp
index 1e9018b4b8..937e635772 100644
--- a/src/test/unit/services/pathfinder/normal_glm_test.cpp
+++ b/src/test/unit/services/pathfinder/normal_glm_test.cpp
@@ -10,6 +10,7 @@
 #include <test/unit/services/instrumented_callbacks.hpp>
 #include <test/unit/services/pathfinder/util.hpp>
 #include <test/unit/services/util.hpp>
+#include <rapidjson/document.h>
 #include <gtest/gtest.h>
 
 // Locally tests can use threads but for jenkins we should just use 1 thread
@@ -131,6 +132,8 @@ TEST_F(ServicesPathfinderGLM, single) {
   for (int i = 2; i < all_mean_vals.cols(); ++i) {
     EXPECT_NEAR(0, all_sd_vals(2, i), .1);
   }
+  rapidjson::Document document;
+  ASSERT_FALSE(document.Parse<0>(diagnostic_ss.str().c_str()).HasParseError());
 }
 
 TEST_F(ServicesPathfinderGLM, multi) {