Skip to content

Commit 7a0da2c

Browse files
abhinav04sharmafacebook-github-bot
abhinav04sharma
authored and
facebook-github-bot
committed
Replacing boost ptree by rapidjson and Better handle bad db_metadata value (#1097)
Summary: Rapidjson is more efficient in JSON parsing. This code was left out while migrating most of trx_meta_data code. Squash with D14652380 Reference Patch: 9f014d31c20 Reference Patch: 145292832b9 Pull Request resolved: #1097 Reviewed By: yizhang82 Differential Revision: D19564342 Pulled By: abhinav04sharma fbshipit-source-id: e930c35
1 parent 98a7f81 commit 7a0da2c

File tree

8 files changed

+118
-59
lines changed

8 files changed

+118
-59
lines changed

‎mysql-test/r/db_metadata.result

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ def test7 {"shard":"test7_shard"}
126126
def test8 NULL
127127
def test9 {"shard":"test9_shard"}
128128
create database test10 db_metadata = "invalid_json";
129-
ERROR HY000: Invalid JSON for DB_METADATA attribute: invalid_json.
129+
ERROR HY000: Invalid JSON object for DB_METADATA attribute: invalid_json.
130130
select catalog_name, schema_name, db_metadata from information_schema.schemata_ext;
131131
CATALOG_NAME SCHEMA_NAME DB_METADATA
132132
def information_schema NULL
@@ -144,7 +144,7 @@ def test7 {"shard":"test7_shard"}
144144
def test8 NULL
145145
def test9 {"shard":"test9_shard"}
146146
create database test11 db_metadata = "{\'shard\':\'test11_shard\'}";
147-
ERROR HY000: Invalid JSON for DB_METADATA attribute: {'shard':'test11_shard'}.
147+
ERROR HY000: Invalid JSON object for DB_METADATA attribute: {'shard':'test11_shard'}.
148148
select catalog_name, schema_name, db_metadata from information_schema.schemata_ext;
149149
CATALOG_NAME SCHEMA_NAME DB_METADATA
150150
def information_schema NULL
@@ -161,6 +161,24 @@ def test6 {"shard":"test6_shard"}
161161
def test7 {"shard":"test7_shard"}
162162
def test8 NULL
163163
def test9 {"shard":"test9_shard"}
164+
CREATE DATABASE test14 db_metadata = "[{\"shard\":\"test14_shard\", \"data\":\"1234\"}]";
165+
ERROR HY000: Invalid JSON object for DB_METADATA attribute: [{"shard":"test14_shard", "data":"1234"}].
166+
SELECT catalog_name, schema_name, db_metadata FROM information_schema.schemata_ext;
167+
CATALOG_NAME SCHEMA_NAME DB_METADATA
168+
def information_schema NULL
169+
def mtr NULL
170+
def mysql NULL
171+
def performance_schema NULL
172+
def sys NULL
173+
def test NULL
174+
def test2 NULL
175+
def test3 NULL
176+
def test4 NULL
177+
def test5 {"shard":"test5_shard"}
178+
def test6 {"shard":"test6_shard"}
179+
def test7 {"shard":"test7_shard"}
180+
def test8 NULL
181+
def test9 {"shard":"test9_shard"}
164182
alter database test3 character set ascii;
165183
show create database test3;
166184
Database Create Database
@@ -366,7 +384,7 @@ def test7 {"shard":"test7_shard_altered"}
366384
def test8 NULL
367385
def test9 {"shard":"test9_shard_altered"}
368386
alter database test9 db_metadata = "invalid_json";
369-
ERROR HY000: Invalid JSON for DB_METADATA attribute: invalid_json.
387+
ERROR HY000: Invalid JSON object for DB_METADATA attribute: invalid_json.
370388
show create database test9;
371389
Database Create Database
372390
test9 CREATE DATABASE `test9` /*!40100 DEFAULT CHARACTER SET ascii READ_ONLY DB_METADATA '{"shard":"test9_shard_altered"}' */ /*!80016 DEFAULT ENCRYPTION='N' */
@@ -386,6 +404,24 @@ def test6 {"shard":"test6_shard_altered"}
386404
def test7 {"shard":"test7_shard_altered"}
387405
def test8 NULL
388406
def test9 {"shard":"test9_shard_altered"}
407+
ALTER DATABASE test10 db_metadata = "[{\"shard\":\"test10_shard\", \"data\":\"1234\"}]";
408+
ERROR HY000: Invalid JSON object for DB_METADATA attribute: [{"shard":"test10_shard", "data":"1234"}].
409+
SELECT catalog_name, schema_name, db_metadata FROM information_schema.schemata_ext;
410+
CATALOG_NAME SCHEMA_NAME DB_METADATA
411+
def information_schema NULL
412+
def mtr NULL
413+
def mysql NULL
414+
def performance_schema NULL
415+
def sys NULL
416+
def test NULL
417+
def test2 NULL
418+
def test3 NULL
419+
def test4 NULL
420+
def test5 {"shard":"Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Really long shard name. Fin"}
421+
def test6 {"shard":"test6_shard_altered"}
422+
def test7 {"shard":"test7_shard_altered"}
423+
def test8 NULL
424+
def test9 {"shard":"test9_shard_altered"}
389425
create database test10;
390426
show create database test10;
391427
Database Create Database
@@ -403,7 +439,7 @@ show create database test10;
403439
Database Create Database
404440
test10 CREATE DATABASE `test10` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DB_METADATA '{"shard":"test10_shard_altered"}' */ /*!80016 DEFAULT ENCRYPTION='N' */
405441
create database test11 db_metadata = "{\'shard\':\'test11_shard\'}";
406-
ERROR HY000: Invalid JSON for DB_METADATA attribute: {'shard':'test11_shard'}.
442+
ERROR HY000: Invalid JSON object for DB_METADATA attribute: {'shard':'test11_shard'}.
407443
create database test12 db_metadata = "{\"sha'rd\":\"test12\\\"_shard\"}";
408444
show create database test12;
409445
Database Create Database
@@ -433,3 +469,6 @@ drop database test9;
433469
drop database test10;
434470
drop database test12;
435471
drop database test13;
472+
DROP DATABASE IF EXISTS test14;
473+
Warnings:
474+
Note 1008 Can't drop database 'test14'; database doesn't exist

‎mysql-test/t/db_metadata.test

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,11 @@ select catalog_name, schema_name, db_metadata from information_schema.schemata_e
5656
create database test11 db_metadata = "{\'shard\':\'test11_shard\'}";
5757
select catalog_name, schema_name, db_metadata from information_schema.schemata_ext;
5858

59+
# create database with JSON array
60+
--error ER_DB_METADATA_INVALID_JSON
61+
CREATE DATABASE test14 db_metadata = "[{\"shard\":\"test14_shard\", \"data\":\"1234\"}]";
62+
SELECT catalog_name, schema_name, db_metadata FROM information_schema.schemata_ext;
63+
5964
# alter database tests
6065

6166
# alter database character set
@@ -132,6 +137,11 @@ alter database test9 db_metadata = "invalid_json";
132137
show create database test9;
133138
select catalog_name, schema_name, db_metadata from information_schema.schemata_ext;
134139

140+
# alter database with JSON array
141+
--error ER_DB_METADATA_INVALID_JSON
142+
ALTER DATABASE test10 db_metadata = "[{\"shard\":\"test10_shard\", \"data\":\"1234\"}]";
143+
SELECT catalog_name, schema_name, db_metadata FROM information_schema.schemata_ext;
144+
135145

136146
# create database without any options
137147
create database test10;
@@ -182,3 +192,4 @@ drop database test9;
182192
drop database test10;
183193
drop database test12;
184194
drop database test13;
195+
DROP DATABASE IF EXISTS test14;

‎share/errmsg-utf8.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19841,7 +19841,7 @@ ER_DB_METADATA_TOO_LONG
1984119841
eng "Metadata for the database is too long. Max length is %d bytes"
1984219842

1984319843
ER_DB_METADATA_INVALID_JSON
19844-
eng "Invalid JSON for DB_METADATA attribute: %s."
19844+
eng "Invalid JSON object for DB_METADATA attribute: %s."
1984519845

1984619846
ER_DB_METADATA_READ_ERROR
1984719847
eng "Error reading db metadata option: '%-.64s'"

‎sql/binlog.cc

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9991,6 +9991,13 @@ std::string THD::gen_trx_metadata() {
99919991
std::string json = buf.GetString();
99929992
boost::trim(json);
99939993

9994+
// verify doc json document
9995+
if (!doc.IsObject()) {
9996+
// NO_LINT_DEBUG
9997+
sql_print_error("Bad JSON format after adding meta data: %s", json.c_str());
9998+
DBUG_RETURN("");
9999+
}
10000+
999410001
std::string comment_str =
999510002
std::string("/*").append(TRX_META_DATA_HEADER).append(json).append("*/");
999610003

@@ -10038,7 +10045,11 @@ bool THD::add_db_metadata(rapidjson::Document &meta_data_root) {
1003810045

1003910046
if (!local_db_metadata.empty()) {
1004010047
rapidjson::Document db_metadata_root;
10041-
if (db_metadata_root.Parse(local_db_metadata.c_str()).HasParseError()) {
10048+
// rapidjson doesn't like calling GetObject() on json non-object value
10049+
// The local_db_metadata format should similar to the following example:
10050+
// {"shard":"<shard_name>", "replicaset":"<replicaset_id>"}
10051+
if (db_metadata_root.Parse(local_db_metadata.c_str()).HasParseError() ||
10052+
!db_metadata_root.IsObject()) {
1004210053
// NO_LINT_DEBUG
1004310054
sql_print_error("Exception while reading meta data: %s",
1004410055
local_db_metadata.c_str());

‎sql/log_event.cc

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,6 @@
4040
#include <string>
4141
#include <utility>
4242

43-
#include <boost/algorithm/string.hpp>
44-
#include <boost/property_tree/json_parser.hpp>
45-
#include <boost/property_tree/ptree.hpp>
46-
4743
#include "base64.h"
4844
#include "binary_log_funcs.h" // my_timestamp_binary_length
4945
#include "debug_vars.h"
@@ -13023,21 +13019,24 @@ void Ignorable_log_event::print(FILE *,
1302313019
}
1302413020
#endif
1302513021

13022+
static const std::string ts_key{"ts"};
13023+
1302613024
ulonglong Rows_query_log_event::extract_last_timestamp() const {
13027-
boost::property_tree::ptree pt;
13025+
rapidjson::Document pt;
1302813026
std::string json = extract_trx_meta_data();
1302913027
if (json.empty()) return 0;
13030-
std::istringstream is(json);
13031-
try {
13032-
read_json(is, pt);
13033-
} catch (const std::exception &e) {
13028+
if (pt.Parse(json.c_str()).HasParseError()) {
1303413029
return 0;
1303513030
}
13036-
13037-
auto timestamps = pt.get_child("ts", boost::property_tree::ptree());
13038-
DBUG_ASSERT(!timestamps.empty());
13039-
return timestamps.empty() ? 0
13040-
: timestamps.back().second.get_value<ulonglong>();
13031+
const auto ts_iter = pt.FindMember(ts_key.c_str());
13032+
ulonglong timestamps = 0;
13033+
if (ts_iter != pt.MemberEnd() && ts_iter->value.IsArray()) {
13034+
for (auto iter = ts_iter->value.MemberBegin();
13035+
iter != ts_iter->value.MemberEnd(); ++iter) {
13036+
timestamps = iter->value.GetUint64();
13037+
}
13038+
}
13039+
return timestamps;
1304113040
}
1304213041

1304313042
Rows_query_log_event::Rows_query_log_event(

‎sql/query_tag_perf_counter.cc

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
// Copyright 2004-present Facebook. All Rights Reserved.
22

3-
#include <boost/lexical_cast.hpp>
4-
#include <boost/optional.hpp>
5-
#include <boost/property_tree/json_parser.hpp>
6-
#include <boost/property_tree/ptree.hpp>
73
#include <deque>
84
#include <memory>
95
#include <mutex>
@@ -17,8 +13,6 @@
1713

1814
namespace qutils {
1915

20-
namespace pt = boost::property_tree;
21-
2216
using cpu_and_num_queries = std::tuple<uint64_t, uint64_t>;
2317
static int64_t timespec_diff(const timespec &start, const timespec &stop);
2418

‎sql/sql_class.cc

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,6 @@
3232
#include <memory>
3333
#include <utility>
3434

35-
#include <boost/lexical_cast.hpp>
36-
#include <boost/optional.hpp>
37-
#include <boost/property_tree/json_parser.hpp>
38-
#include <boost/property_tree/ptree.hpp>
3935
#include <sstream>
4036

4137
#include "m_ctype.h"
@@ -102,8 +98,6 @@
10298
#include "sql/xa.h"
10399
#include "thr_mutex.h"
104100

105-
using boost::property_tree::ptree;
106-
107101
using std::max;
108102
using std::min;
109103
using std::unique_ptr;
@@ -3048,29 +3042,44 @@ void THD::set_query_attrs(const char *attrs, size_t length) {
30483042
mysql_mutex_unlock(&LOCK_thd_data);
30493043
}
30503044

3051-
int THD::parse_query_info_attr() {
3052-
static const std::string query_info_key = "query_info";
3053-
static const std::string traceid_key = "traceid";
3045+
static const std::string query_info_key{"query_info"};
3046+
static const std::string traceid_key{"traceid"};
3047+
static const std::string query_type_key{"query_type"};
3048+
static const std::string num_queries_key{"num_queries"};
30543049

3050+
int THD::parse_query_info_attr() {
30553051
for (const auto &kvp : query_attrs_list) {
30563052
if (kvp.first == traceid_key) {
30573053
this->trace_id = kvp.second;
30583054
} else if (kvp.first == query_info_key) {
3059-
ptree root;
3060-
try {
3061-
std::istringstream query_info_attr(kvp.second);
3062-
boost::property_tree::read_json(query_info_attr, root);
3063-
} catch (const boost::property_tree::json_parser::json_parser_error &e) {
3064-
return -1; // invalid json
3055+
rapidjson::Document root;
3056+
if (root.Parse(kvp.second.c_str()).HasParseError()) {
3057+
return -1;
3058+
}
3059+
3060+
{
3061+
const auto iter = root.FindMember(traceid_key.c_str());
3062+
if (iter != root.MemberEnd()) {
3063+
this->trace_id = iter->value.GetString();
3064+
}
3065+
}
3066+
3067+
{
3068+
const auto iter = root.FindMember(query_type_key.c_str());
3069+
if (iter != root.MemberEnd()) {
3070+
this->query_type = iter->value.GetString();
3071+
} else {
3072+
return -1;
3073+
}
30653074
}
3066-
try {
3067-
boost::optional<std::string> trace_id =
3068-
root.get_optional<std::string>("traceid");
3069-
if (trace_id) this->trace_id = *trace_id;
3070-
this->query_type = root.get<std::string>("query_type");
3071-
this->num_queries = root.get<uint64_t>("num_queries");
3072-
} catch (const boost::property_tree::ptree_error &e) {
3073-
return -1; // invalid key or value
3075+
3076+
{
3077+
const auto iter = root.FindMember(num_queries_key.c_str());
3078+
if (iter != root.MemberEnd()) {
3079+
this->num_queries = iter->value.GetUint64();
3080+
} else {
3081+
return -1;
3082+
}
30743083
}
30753084
return 0;
30763085
}

‎sql/sql_yacc.yy

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,9 @@ Note: YYTHD is passed as an argument to yyparse(), and subsequently to yylex().
4646
#include <limits>
4747
#include <type_traits> // for std::remove_reference
4848
#include <sstream>
49-
#include <boost/property_tree/ptree.hpp>
50-
#include <boost/property_tree/json_parser.hpp>
49+
#include "my_rapidjson_size_t.h"
50+
#include "rapidjson/document.h"
51+
#include "rapidjson/writer.h"
5152

5253
#include "my_dbug.h"
5354
#include "myisam.h"
@@ -6203,14 +6204,9 @@ db_metadata_str:
62036204
/* Verify that a valid JSON is provided */
62046205
if ($3.length > 0)
62056206
{
6206-
boost::property_tree::ptree db_metadata_root;
6207-
std::istringstream is($3.str);
6208-
try
6209-
{
6210-
boost::property_tree::json_parser::read_json(is,
6211-
db_metadata_root);
6212-
}
6213-
catch (const std::exception&)
6207+
rapidjson::Document db_metadata_root;
6208+
if (db_metadata_root.Parse($3.str).HasParseError() ||
6209+
!db_metadata_root.IsObject())
62146210
{
62156211
my_error(ER_DB_METADATA_INVALID_JSON, MYF(0), $3.str);
62166212
MYSQL_YYABORT;

0 commit comments

Comments
 (0)