Replacing boost ptree by rapidjson and Better handle bad db_metadata …

…value (#1097)

Summary:
Rapidjson is more efficient in JSON parsing. This code was left out
while migrating most of trx_meta_data code.

Squash with D14652380

Reference Patch: 9f014d31c20
Reference Patch: 145292832b9

Pull Request resolved: #1097

Reviewed By: yizhang82

Differential Revision: D19564342

Pulled By: abhinav04sharma

fbshipit-source-id: e930c35

mysql-test/r/db_metadata.result

@@ -126,7 +126,7 @@ def	test7	{"shard":"test7_shard"}
 def	test8	NULL
 def	test9	{"shard":"test9_shard"}
 create database test10 db_metadata = "invalid_json";
-ERROR HY000: Invalid JSON for DB_METADATA attribute: invalid_json.
+ERROR HY000: Invalid JSON object for DB_METADATA attribute: invalid_json.
 select catalog_name, schema_name, db_metadata from information_schema.schemata_ext;
 CATALOG_NAME	SCHEMA_NAME	DB_METADATA
 def	information_schema	NULL
@@ -144,7 +144,7 @@ def	test7	{"shard":"test7_shard"}
 def	test8	NULL
 def	test9	{"shard":"test9_shard"}
 create database test11 db_metadata = "{\'shard\':\'test11_shard\'}";
-ERROR HY000: Invalid JSON for DB_METADATA attribute: {'shard':'test11_shard'}.
+ERROR HY000: Invalid JSON object for DB_METADATA attribute: {'shard':'test11_shard'}.
 select catalog_name, schema_name, db_metadata from information_schema.schemata_ext;
 CATALOG_NAME	SCHEMA_NAME	DB_METADATA
 def	information_schema	NULL
@@ -161,6 +161,24 @@ def	test6	{"shard":"test6_shard"}
 def	test7	{"shard":"test7_shard"}
 def	test8	NULL
 def	test9	{"shard":"test9_shard"}
+CREATE DATABASE test14 db_metadata = "[{\"shard\":\"test14_shard\", \"data\":\"1234\"}]";
+ERROR HY000: Invalid JSON object for DB_METADATA attribute: [{"shard":"test14_shard", "data":"1234"}].
+SELECT catalog_name, schema_name, db_metadata FROM information_schema.schemata_ext;
+CATALOG_NAME	SCHEMA_NAME	DB_METADATA
+def	information_schema	NULL
+def	mtr	NULL
+def	mysql	NULL
+def	performance_schema	NULL
+def	sys	NULL
+def	test	NULL
+def	test2	NULL
+def	test3	NULL
+def	test4	NULL
+def	test5	{"shard":"test5_shard"}
+def	test6	{"shard":"test6_shard"}
+def	test7	{"shard":"test7_shard"}
+def	test8	NULL
+def	test9	{"shard":"test9_shard"}
 alter database test3 character set ascii;
 show create database test3;
 Database	Create Database
@@ -366,7 +384,7 @@ def	test7	{"shard":"test7_shard_altered"}
 def	test8	NULL
 def	test9	{"shard":"test9_shard_altered"}
 alter database test9 db_metadata = "invalid_json";
-ERROR HY000: Invalid JSON for DB_METADATA attribute: invalid_json.
+ERROR HY000: Invalid JSON object for DB_METADATA attribute: invalid_json.
 show create database test9;
 Database	Create Database
 test9	CREATE DATABASE `test9` /*!40100 DEFAULT CHARACTER SET ascii READ_ONLY DB_METADATA '{"shard":"test9_shard_altered"}' */ /*!80016 DEFAULT ENCRYPTION='N' */
@@ -386,6 +404,24 @@ def	test6	{"shard":"test6_shard_altered"}
 def	test7	{"shard":"test7_shard_altered"}
 def	test8	NULL
 def	test9	{"shard":"test9_shard_altered"}
+ALTER DATABASE test10 db_metadata = "[{\"shard\":\"test10_shard\", \"data\":\"1234\"}]";
+ERROR HY000: Invalid JSON object for DB_METADATA attribute: [{"shard":"test10_shard", "data":"1234"}].
+SELECT catalog_name, schema_name, db_metadata FROM information_schema.schemata_ext;
+CATALOG_NAME	SCHEMA_NAME	DB_METADATA
+def	information_schema	NULL
+def	mtr	NULL
+def	mysql	NULL
+def	performance_schema	NULL
+def	sys	NULL
+def	test	NULL
+def	test2	NULL
+def	test3	NULL
+def	test4	NULL
+def	test5	{"shard":"Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Fin"}
+def	test6	{"shard":"test6_shard_altered"}
+def	test7	{"shard":"test7_shard_altered"}
+def	test8	NULL
+def	test9	{"shard":"test9_shard_altered"}
 create database test10;
 show create database test10;
 Database	Create Database
@@ -403,7 +439,7 @@ show create database test10;
 Database	Create Database
 test10	CREATE DATABASE `test10` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DB_METADATA '{"shard":"test10_shard_altered"}' */ /*!80016 DEFAULT ENCRYPTION='N' */
 create database test11 db_metadata = "{\'shard\':\'test11_shard\'}";
-ERROR HY000: Invalid JSON for DB_METADATA attribute: {'shard':'test11_shard'}.
+ERROR HY000: Invalid JSON object for DB_METADATA attribute: {'shard':'test11_shard'}.
 create database test12 db_metadata = "{\"sha'rd\":\"test12\\\"_shard\"}";
 show create database test12;
 Database	Create Database
@@ -433,3 +469,6 @@ drop database test9;
 drop database test10;
 drop database test12;
 drop database test13;
+DROP DATABASE IF EXISTS test14;
+Warnings:
+Note	1008	Can't drop database 'test14'; database doesn't exist

mysql-test/t/db_metadata.test

@@ -56,6 +56,11 @@ select catalog_name, schema_name, db_metadata from information_schema.schemata_e
 create database test11 db_metadata = "{\'shard\':\'test11_shard\'}";
 select catalog_name, schema_name, db_metadata from information_schema.schemata_ext;
 
+# create database with JSON array
+--error ER_DB_METADATA_INVALID_JSON
+CREATE DATABASE test14 db_metadata = "[{\"shard\":\"test14_shard\", \"data\":\"1234\"}]";
+SELECT catalog_name, schema_name, db_metadata FROM information_schema.schemata_ext;
+
 # alter database tests
 
 # alter database character set
@@ -132,6 +137,11 @@ alter database test9 db_metadata = "invalid_json";
 show create database test9;
 select catalog_name, schema_name, db_metadata from information_schema.schemata_ext;
 
+# alter database with JSON array
+--error ER_DB_METADATA_INVALID_JSON
+ALTER DATABASE test10 db_metadata = "[{\"shard\":\"test10_shard\", \"data\":\"1234\"}]";
+SELECT catalog_name, schema_name, db_metadata FROM information_schema.schemata_ext;
+
 
 # create database without any options
 create database test10;
@@ -182,3 +192,4 @@ drop database test9;
 drop database test10;
 drop database test12;
 drop database test13;
+DROP DATABASE IF EXISTS test14;

share/errmsg-utf8.txt

@@ -19841,7 +19841,7 @@ ER_DB_METADATA_TOO_LONG
   eng "Metadata for the database is too long. Max length is %d bytes"
 
 ER_DB_METADATA_INVALID_JSON
-  eng "Invalid JSON for DB_METADATA attribute: %s."
+  eng "Invalid JSON object for DB_METADATA attribute: %s."
 
 ER_DB_METADATA_READ_ERROR
   eng "Error reading db metadata option: '%-.64s'"

sql/binlog.cc

@@ -9991,6 +9991,13 @@ std::string THD::gen_trx_metadata() {
   std::string json = buf.GetString();
   boost::trim(json);
 
+  // verify doc json document
+  if (!doc.IsObject()) {
+    // NO_LINT_DEBUG
+    sql_print_error("Bad JSON format after adding meta data: %s", json.c_str());
+    DBUG_RETURN("");
+  }
+
   std::string comment_str =
       std::string("/*").append(TRX_META_DATA_HEADER).append(json).append("*/");
 
@@ -10038,7 +10045,11 @@ bool THD::add_db_metadata(rapidjson::Document &meta_data_root) {
 
   if (!local_db_metadata.empty()) {
     rapidjson::Document db_metadata_root;
-    if (db_metadata_root.Parse(local_db_metadata.c_str()).HasParseError()) {
+    // rapidjson doesn't like calling GetObject() on json non-object value
+    // The local_db_metadata format should similar to the following example:
+    // {"shard":"<shard_name>", "replicaset":"<replicaset_id>"}
+    if (db_metadata_root.Parse(local_db_metadata.c_str()).HasParseError() ||
+        !db_metadata_root.IsObject()) {
       // NO_LINT_DEBUG
       sql_print_error("Exception while reading meta data: %s",
                       local_db_metadata.c_str());

sql/log_event.cc

@@ -40,10 +40,6 @@
 #include <string>
 #include <utility>
 
-#include <boost/algorithm/string.hpp>
-#include <boost/property_tree/json_parser.hpp>
-#include <boost/property_tree/ptree.hpp>
-
 #include "base64.h"
 #include "binary_log_funcs.h"  // my_timestamp_binary_length
 #include "debug_vars.h"
@@ -13023,21 +13019,24 @@ void Ignorable_log_event::print(FILE *,
 }
 #endif
 
+static const std::string ts_key{"ts"};
+
 ulonglong Rows_query_log_event::extract_last_timestamp() const {
-  boost::property_tree::ptree pt;
+  rapidjson::Document pt;
   std::string json = extract_trx_meta_data();
   if (json.empty()) return 0;
-  std::istringstream is(json);
-  try {
-    read_json(is, pt);
-  } catch (const std::exception &e) {
+  if (pt.Parse(json.c_str()).HasParseError()) {
     return 0;
   }
-
-  auto timestamps = pt.get_child("ts", boost::property_tree::ptree());
-  DBUG_ASSERT(!timestamps.empty());
-  return timestamps.empty() ? 0
-                            : timestamps.back().second.get_value<ulonglong>();
+  const auto ts_iter = pt.FindMember(ts_key.c_str());
+  ulonglong timestamps = 0;
+  if (ts_iter != pt.MemberEnd() && ts_iter->value.IsArray()) {
+    for (auto iter = ts_iter->value.MemberBegin();
+         iter != ts_iter->value.MemberEnd(); ++iter) {
+      timestamps = iter->value.GetUint64();
+    }
+  }
+  return timestamps;
 }
 
 Rows_query_log_event::Rows_query_log_event(

sql/query_tag_perf_counter.cc

@@ -1,9 +1,5 @@
 // Copyright 2004-present Facebook. All Rights Reserved.
 
-#include <boost/lexical_cast.hpp>
-#include <boost/optional.hpp>
-#include <boost/property_tree/json_parser.hpp>
-#include <boost/property_tree/ptree.hpp>
 #include <deque>
 #include <memory>
 #include <mutex>
@@ -17,8 +13,6 @@
 
 namespace qutils {
 
-namespace pt = boost::property_tree;
-
 using cpu_and_num_queries = std::tuple<uint64_t, uint64_t>;
 static int64_t timespec_diff(const timespec &start, const timespec &stop);
 

sql/sql_class.cc

@@ -32,10 +32,6 @@
 #include <memory>
 #include <utility>
 
-#include <boost/lexical_cast.hpp>
-#include <boost/optional.hpp>
-#include <boost/property_tree/json_parser.hpp>
-#include <boost/property_tree/ptree.hpp>
 #include <sstream>
 
 #include "m_ctype.h"
@@ -102,8 +98,6 @@
 #include "sql/xa.h"
 #include "thr_mutex.h"
 
-using boost::property_tree::ptree;
-
 using std::max;
 using std::min;
 using std::unique_ptr;
@@ -3048,29 +3042,44 @@ void THD::set_query_attrs(const char *attrs, size_t length) {
   mysql_mutex_unlock(&LOCK_thd_data);
 }
 
-int THD::parse_query_info_attr() {
-  static const std::string query_info_key = "query_info";
-  static const std::string traceid_key = "traceid";
+static const std::string query_info_key{"query_info"};
+static const std::string traceid_key{"traceid"};
+static const std::string query_type_key{"query_type"};
+static const std::string num_queries_key{"num_queries"};
 
+int THD::parse_query_info_attr() {
   for (const auto &kvp : query_attrs_list) {
     if (kvp.first == traceid_key) {
       this->trace_id = kvp.second;
     } else if (kvp.first == query_info_key) {
-      ptree root;
-      try {
-        std::istringstream query_info_attr(kvp.second);
-        boost::property_tree::read_json(query_info_attr, root);
-      } catch (const boost::property_tree::json_parser::json_parser_error &e) {
-        return -1;  // invalid json
+      rapidjson::Document root;
+      if (root.Parse(kvp.second.c_str()).HasParseError()) {
+        return -1;
+      }
+
+      {
+        const auto iter = root.FindMember(traceid_key.c_str());
+        if (iter != root.MemberEnd()) {
+          this->trace_id = iter->value.GetString();
+        }
+      }
+
+      {
+        const auto iter = root.FindMember(query_type_key.c_str());
+        if (iter != root.MemberEnd()) {
+          this->query_type = iter->value.GetString();
+        } else {
+          return -1;
+        }
       }
-      try {
-        boost::optional<std::string> trace_id =
-            root.get_optional<std::string>("traceid");
-        if (trace_id) this->trace_id = *trace_id;
-        this->query_type = root.get<std::string>("query_type");
-        this->num_queries = root.get<uint64_t>("num_queries");
-      } catch (const boost::property_tree::ptree_error &e) {
-        return -1;  // invalid key or value
+
+      {
+        const auto iter = root.FindMember(num_queries_key.c_str());
+        if (iter != root.MemberEnd()) {
+          this->num_queries = iter->value.GetUint64();
+        } else {
+          return -1;
+        }
       }
       return 0;
     }

sql/sql_yacc.yy

@@ -46,8 +46,9 @@ Note: YYTHD is passed as an argument to yyparse(), and subsequently to yylex().
 #include <limits>
 #include <type_traits>                       // for std::remove_reference
 #include <sstream>
-#include <boost/property_tree/ptree.hpp>
-#include <boost/property_tree/json_parser.hpp>
+#include "my_rapidjson_size_t.h"
+#include "rapidjson/document.h"
+#include "rapidjson/writer.h"
 
 #include "my_dbug.h"
 #include "myisam.h"
@@ -6203,14 +6204,9 @@ db_metadata_str:
             /* Verify that a valid JSON is provided */
             if ($3.length > 0)
             {
-              boost::property_tree::ptree db_metadata_root;
-              std::istringstream is($3.str);
-              try
-              {
-                boost::property_tree::json_parser::read_json(is,
-                                                             db_metadata_root);
-              }
-              catch (const std::exception&)
+              rapidjson::Document db_metadata_root;
+              if (db_metadata_root.Parse($3.str).HasParseError() ||
+                  !db_metadata_root.IsObject())
               {
                 my_error(ER_DB_METADATA_INVALID_JSON, MYF(0), $3.str);
                 MYSQL_YYABORT;