Skip to content

[fix](function)fix get_json_string does not support array #59328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
y115520509 wants to merge 1 commit into
base: branch-2.1
Choose a base branch
from
Open

[fix](function)fix get_json_string does not support array #59328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 38 additions & 1 deletion be/src/vec/functions/function_json.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,43 @@ rapidjson::Value* match_value(const std::vector<JsonPath>& parsed_paths, rapidjs
} else {
root = &((*root)[col.c_str()]);
}
} else if (root->IsArray()) {
array_obj = static_cast<rapidjson::Value*>(
mem_allocator.Malloc(sizeof(rapidjson::Value)));
array_obj->SetArray();
bool is_null = true;

// if array ,loop the array,find out all Objects,then find the results from the objects
for (int j = 0; j < root->Size(); j++) {
rapidjson::Value* json_elem = &((*root)[j]);

if (json_elem->IsArray() || json_elem->IsNull()) {
continue;
} else {
if (!json_elem->IsObject()) {
continue;
}
if (!json_elem->HasMember(col.c_str())) {
if (is_insert_null) { // not found item, then insert a null object.
is_null = false;
rapidjson::Value nullObject(rapidjson::kNullType);
array_obj->PushBack(nullObject, mem_allocator);
}
continue;
}
rapidjson::Value* obj = &((*json_elem)[col.c_str()]);
if (obj->IsArray()) {
is_null = false;
for (int k = 0; k < obj->Size(); k++) {
array_obj->PushBack((*obj)[k], mem_allocator);
}
} else if (!obj->IsNull()) {
is_null = false;
array_obj->PushBack(*obj, mem_allocator);
}
}
}
root = is_null ? nullptr : array_obj;
} else {
// root is not a nested type, return NULL
return nullptr;
Expand All @@ -171,7 +208,7 @@ rapidjson::Value* match_value(const std::vector<JsonPath>& parsed_paths, rapidjs
if (UNLIKELY(index != -1)) {
// judge the rapidjson:Value, which base the top's result,
// if not array return NULL;else get the index value from the array
if (root->IsArray()) {
if (NULL != root && root->IsArray()) {
if (root->IsNull()) {
return nullptr;
} else if (index == -2) {
Expand Down
2 changes: 1 addition & 1 deletion be/test/vec/function/function_json_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ TEST(FunctionJsonTEST, GetJsonStringTest) {
{{VARCHAR(R"({"k1.key":{"k2":["v1", "v2"]}})"), VARCHAR("$.\"k1.key\".k2[0]")},
VARCHAR("v1")},
{{VARCHAR(R"([{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}])"), VARCHAR("$.k1")},
Null()}};
VARCHAR(R"(["v1","v3","v4"])")}};

static_cast<void>(check_function<DataTypeString, true>(func_name, input_types, data_set));
}
Expand Down
12 changes: 6 additions & 6 deletions regression-test/data/json_p0/test_json_load_and_function.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6189,7 +6189,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31"
26 \N \N
27 {"k1":"v1","k2":200} "v1"
Expand Down Expand Up @@ -6218,7 +6218,7 @@ false
14 [123,456] [456]
15 ["abc","def"] ["def"]
16 [null,true,false,100,6.18,"abc"] [true]
17 [{"k1":"v41","k2":400},1,"a",3.14] [1]
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
26 \N \N
27 {"k1":"v1","k2":200} [200]
Expand Down Expand Up @@ -6247,7 +6247,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
26 \N \N
27 {"k1":"v1","k2":200} [200]
Expand Down Expand Up @@ -11664,7 +11664,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31"
27 {"k1":"v1","k2":200} "v1"
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
Expand All @@ -11691,7 +11691,7 @@ false
14 [123,456] [456]
15 ["abc","def"] ["def"]
16 [null,true,false,100,6.18,"abc"] [true]
17 [{"k1":"v41","k2":400},1,"a",3.14] [1]
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
27 {"k1":"v1","k2":200} [200]
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
Expand All @@ -11718,7 +11718,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
27 {"k1":"v1","k2":200} [200]
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
Expand Down
6 changes: 3 additions & 3 deletions regression-test/data/jsonb_p0/test_jsonb_load_and_function.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8187,7 +8187,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31"
26 \N \N
27 {"k1":"v1","k2":200} "v1"
Expand Down Expand Up @@ -8216,7 +8216,7 @@ false
14 [123,456] [456]
15 ["abc","def"] ["def"]
16 [null,true,false,100,6.18,"abc"] [true]
17 [{"k1":"v41","k2":400},1,"a",3.14] [1]
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
26 \N \N
27 {"k1":"v1","k2":200} [200]
Expand Down Expand Up @@ -8245,7 +8245,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
26 \N \N
27 {"k1":"v1","k2":200} [200]
Expand Down
6 changes: 3 additions & 3 deletions regression-test/data/nereids_function_p0/scalar_function/J.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7348,7 +7348,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31"
26 \N \N
27 {"k1":"v1","k2":200} "v1"
Expand All @@ -7374,7 +7374,7 @@ false
14 [123,456] [456]
15 ["abc","def"] ["def"]
16 [null,true,false,100,6.18,"abc"] [true]
17 [{"k1":"v41","k2":400},1,"a",3.14] [1]
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
26 \N \N
27 {"k1":"v1","k2":200} [200]
Expand All @@ -7400,7 +7400,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
26 \N \N
27 {"k1":"v1","k2":200} [200]
Expand Down
6 changes: 3 additions & 3 deletions regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6186,7 +6186,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31"
26 \N \N
27 {"k1":"v1","k2":200} "v1"
Expand Down Expand Up @@ -6215,7 +6215,7 @@ false
14 [123,456] [456]
15 ["abc","def"] ["def"]
16 [null,true,false,100,6.18,"abc"] [true]
17 [{"k1":"v41","k2":400},1,"a",3.14] [1]
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
26 \N \N
27 {"k1":"v1","k2":200} [200]
Expand Down Expand Up @@ -6244,7 +6244,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
26 \N \N
27 {"k1":"v1","k2":200} [200]
Expand Down
6 changes: 3 additions & 3 deletions regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8187,7 +8187,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31"
26 \N \N
27 {"k1":"v1","k2":200} "v1"
Expand Down Expand Up @@ -8216,7 +8216,7 @@ false
14 [123,456] [456]
15 ["abc","def"] ["def"]
16 [null,true,false,100,6.18,"abc"] [true]
17 [{"k1":"v41","k2":400},1,"a",3.14] [1]
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
26 \N \N
27 {"k1":"v1","k2":200} [200]
Expand Down Expand Up @@ -8245,7 +8245,7 @@ false
14 [123,456] \N
15 ["abc","def"] \N
16 [null,true,false,100,6.18,"abc"] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] \N
17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]]
18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300]
26 \N \N
27 {"k1":"v1","k2":200} [200]
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/auth_up_down_p0/load.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,6 @@ suite("test_upgrade_downgrade_prepare_auth","p0,auth,restart_fe") {
sql """GRANT USAGE_PRIV ON RESOURCE ${rg1} TO ${user1};"""
connect(user1, "${pwd}", context.config.jdbcUrl) {
def res = sql """SHOW RESOURCES;"""
assertTrue(res.size == 10)
assertTrue(res.size() == 10)
}
}
2 changes: 1 addition & 1 deletion regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,6 @@ suite("test_upgrade_downgrade_compatibility_auth","p0,auth,restart_fe") {
// resource group
connect(user1, "${pwd}", context.config.jdbcUrl) {
def res = sql """SHOW RESOURCES;"""
assertTrue(res.size == 10)
assertTrue(res.size() == 10)
}
}
1 change: 1 addition & 0 deletions ...ession-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ suite("test_iceberg_hadoop_catalog_kerberos", "p0,external,kerberos,external_doc
"""
def table = sql """ show tables like '%${test_tbl_name}' """
assert table.size() == 1

sql """
insert into ${test_tbl_name} values (
'2024-05-26 12:34:56',
Expand Down
10 changes: 5 additions & 5 deletions regression-test/suites/external_table_p0/tvf/test_backends_tvf.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,30 +19,30 @@
suite("test_backends_tvf","p0,external,tvf,external_docker") {
List<List<Object>> table = sql """ select * from backends(); """
assertTrue(table.size() > 0)
assertEquals(25, table[0].size)
assertEquals(25, table[0].size())

// filter columns
table = sql """ select BackendId, Host, Alive, TotalCapacity, Version, NodeRole from backends();"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 6)
assertTrue(table[0].size() == 6)
assertEquals(true, table[0][2])

// case insensitive
table = sql """ select backendid, Host, alive, Totalcapacity, version, nodeRole from backends();"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 6)
assertTrue(table[0].size() == 6)
assertEquals(true, table[0][2])

// test aliase columns
table = sql """ select backendid as id, Host as name, alive, NodeRole as r from backends();"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 4)
assertTrue(table[0].size() == 4)
assertEquals(true, table[0][2])

// test changing position of columns
table = sql """ select Host as name, NodeRole as r, alive from backends();"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 3)
assertTrue(table[0].size() == 3)
assertEquals(true, table[0][2])

def res = sql """ select count(*) from backends() where alive = 1; """
Expand Down
8 changes: 4 additions & 4 deletions regression-test/suites/external_table_p0/tvf/test_frontends_disks_tvf.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,23 @@
suite("test_frontends_disks_tvf", "p0,external,external_docker") {
List<List<Object>> table = sql """ select * from `frontends_disks`(); """
assertTrue(table.size() > 0)
assertTrue(table[0].size == 10)
assertTrue(table[0].size() == 10)

// filter columns
table = sql """ select Name from `frontends_disks`();"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 1)
assertTrue(table[0].size() == 1)

// case insensitive
table = sql """ select name, host, dirtype, dir from frontends_disks() order by dirtype;"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 4)
assertTrue(table[0].size() == 4)
assertEquals("audit-log", table[0][2])

// test aliase columns
table = sql """ select name as n, host as h, dirtype as a from frontends_disks() order by dirtype; """
assertTrue(table.size() > 0)
assertTrue(table[0].size == 3)
assertTrue(table[0].size() == 3)
assertEquals("audit-log", table[0][2])

// test changing position of columns
Expand Down
8 changes: 4 additions & 4 deletions regression-test/suites/external_table_p0/tvf/test_frontends_tvf.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,23 @@
suite("test_frontends_tvf","p0,external,tvf,external_docker") {
List<List<Object>> table = sql """ select * from `frontends`(); """
assertTrue(table.size() > 0)
assertTrue(table[0].size == 19)
assertTrue(table[0].size() == 19)

// filter columns
table = sql """ select Name from `frontends`();"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 1)
assertTrue(table[0].size() == 1)

// case insensitive
table = sql """ select name, host, editlogport, httpport, alive from frontends();"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 5)
assertTrue(table[0].size() == 5)
assertEquals("true", table[0][4])

// test aliase columns
table = sql """ select name as n, host as h, alive as a, editlogport as e from frontends(); """
assertTrue(table.size() > 0)
assertTrue(table[0].size == 4)
assertTrue(table[0].size() == 4)
assertEquals("true", table[0][2])

// test changing position of columns
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/load_p0/stream_load/test_stream_load.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1617,7 +1617,7 @@ suite("test_stream_load", "p0") {

log.info(sql_result[0][0].toString())
log.info(sql_result[0][1].toString())
log.info(sql_result[0].size.toString())
log.info(sql_result[0].size().toString())

def beHost=sql_result[0][0]
def beHttpPort=sql_result[0][1]
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/nereids_syntax_p0/information_schema.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
suite("information_schema") {
List<List<Object>> table = sql """ select * from backends(); """
assertTrue(table.size() > 0)
assertTrue(table[0].size == 25)
assertTrue(table[0].size() == 25)

sql "SELECT DATABASE();"
sql "select USER();"
Expand Down
Loading