From ee803cbbb97bc47a2f6f3f4b80e554cd258e188b Mon Sep 17 00:00:00 2001 From: work Date: Mon, 8 Dec 2025 16:06:39 +0800 Subject: [PATCH] fixed a bug in the get_json_string function for array types. --- be/src/vec/functions/function_json.cpp | 39 ++++++++++++++++++- be/test/vec/function/function_json_test.cpp | 2 +- .../json_p0/test_json_load_and_function.out | 12 +++--- .../jsonb_p0/test_jsonb_load_and_function.out | 6 +-- .../nereids_function_p0/scalar_function/J.out | 6 +-- .../json_p0/test_json_load_and_function.out | 6 +-- .../jsonb_p0/test_jsonb_load_and_function.out | 6 +-- .../suites/auth_up_down_p0/load.groovy | 2 +- .../test_grant_revoke_auth.groovy | 2 +- ...est_iceberg_hadoop_catalog_kerberos.groovy | 1 + .../tvf/test_backends_tvf.groovy | 10 ++--- .../tvf/test_frontends_disks_tvf.groovy | 8 ++-- .../tvf/test_frontends_tvf.groovy | 8 ++-- .../stream_load/test_stream_load.groovy | 2 +- .../information_schema.groovy | 2 +- 15 files changed, 75 insertions(+), 37 deletions(-) diff --git a/be/src/vec/functions/function_json.cpp b/be/src/vec/functions/function_json.cpp index 8299ba14ee8e41..eac7ad9cffb7b3 100644 --- a/be/src/vec/functions/function_json.cpp +++ b/be/src/vec/functions/function_json.cpp @@ -162,6 +162,43 @@ rapidjson::Value* match_value(const std::vector& parsed_paths, rapidjs } else { root = &((*root)[col.c_str()]); } + } else if (root->IsArray()) { + array_obj = static_cast( + mem_allocator.Malloc(sizeof(rapidjson::Value))); + array_obj->SetArray(); + bool is_null = true; + + // if array ,loop the array,find out all Objects,then find the results from the objects + for (int j = 0; j < root->Size(); j++) { + rapidjson::Value* json_elem = &((*root)[j]); + + if (json_elem->IsArray() || json_elem->IsNull()) { + continue; + } else { + if (!json_elem->IsObject()) { + continue; + } + if (!json_elem->HasMember(col.c_str())) { + if (is_insert_null) { // not found item, then insert a null object. + is_null = false; + rapidjson::Value nullObject(rapidjson::kNullType); + array_obj->PushBack(nullObject, mem_allocator); + } + continue; + } + rapidjson::Value* obj = &((*json_elem)[col.c_str()]); + if (obj->IsArray()) { + is_null = false; + for (int k = 0; k < obj->Size(); k++) { + array_obj->PushBack((*obj)[k], mem_allocator); + } + } else if (!obj->IsNull()) { + is_null = false; + array_obj->PushBack(*obj, mem_allocator); + } + } + } + root = is_null ? nullptr : array_obj; } else { // root is not a nested type, return NULL return nullptr; @@ -171,7 +208,7 @@ rapidjson::Value* match_value(const std::vector& parsed_paths, rapidjs if (UNLIKELY(index != -1)) { // judge the rapidjson:Value, which base the top's result, // if not array return NULL;else get the index value from the array - if (root->IsArray()) { + if (NULL != root && root->IsArray()) { if (root->IsNull()) { return nullptr; } else if (index == -2) { diff --git a/be/test/vec/function/function_json_test.cpp b/be/test/vec/function/function_json_test.cpp index 9c79b712463709..bafabcaf3bde18 100644 --- a/be/test/vec/function/function_json_test.cpp +++ b/be/test/vec/function/function_json_test.cpp @@ -78,7 +78,7 @@ TEST(FunctionJsonTEST, GetJsonStringTest) { {{VARCHAR(R"({"k1.key":{"k2":["v1", "v2"]}})"), VARCHAR("$.\"k1.key\".k2[0]")}, VARCHAR("v1")}, {{VARCHAR(R"([{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}])"), VARCHAR("$.k1")}, - Null()}}; + VARCHAR(R"(["v1","v3","v4"])")}}; static_cast(check_function(func_name, input_types, data_set)); } diff --git a/regression-test/data/json_p0/test_json_load_and_function.out b/regression-test/data/json_p0/test_json_load_and_function.out index 9928a5943abb4c..893a95768efa19 100644 --- a/regression-test/data/json_p0/test_json_load_and_function.out +++ b/regression-test/data/json_p0/test_json_load_and_function.out @@ -6189,7 +6189,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31" 26 \N \N 27 {"k1":"v1","k2":200} "v1" @@ -6218,7 +6218,7 @@ false 14 [123,456] [456] 15 ["abc","def"] ["def"] 16 [null,true,false,100,6.18,"abc"] [true] -17 [{"k1":"v41","k2":400},1,"a",3.14] [1] +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N 27 {"k1":"v1","k2":200} [200] @@ -6247,7 +6247,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N 27 {"k1":"v1","k2":200} [200] @@ -11664,7 +11664,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31" 27 {"k1":"v1","k2":200} "v1" 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -11691,7 +11691,7 @@ false 14 [123,456] [456] 15 ["abc","def"] ["def"] 16 [null,true,false,100,6.18,"abc"] [true] -17 [{"k1":"v41","k2":400},1,"a",3.14] [1] +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 27 {"k1":"v1","k2":200} [200] 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -11718,7 +11718,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 27 {"k1":"v1","k2":200} [200] 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out index 773061f7bda5d4..9b40885e65f818 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out @@ -8187,7 +8187,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31" 26 \N \N 27 {"k1":"v1","k2":200} "v1" @@ -8216,7 +8216,7 @@ false 14 [123,456] [456] 15 ["abc","def"] ["def"] 16 [null,true,false,100,6.18,"abc"] [true] -17 [{"k1":"v41","k2":400},1,"a",3.14] [1] +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N 27 {"k1":"v1","k2":200} [200] @@ -8245,7 +8245,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N 27 {"k1":"v1","k2":200} [200] diff --git a/regression-test/data/nereids_function_p0/scalar_function/J.out b/regression-test/data/nereids_function_p0/scalar_function/J.out index dc1c34eafe65f6..4bf420950857fb 100644 --- a/regression-test/data/nereids_function_p0/scalar_function/J.out +++ b/regression-test/data/nereids_function_p0/scalar_function/J.out @@ -7348,7 +7348,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31" 26 \N \N 27 {"k1":"v1","k2":200} "v1" @@ -7374,7 +7374,7 @@ false 14 [123,456] [456] 15 ["abc","def"] ["def"] 16 [null,true,false,100,6.18,"abc"] [true] -17 [{"k1":"v41","k2":400},1,"a",3.14] [1] +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N 27 {"k1":"v1","k2":200} [200] @@ -7400,7 +7400,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N 27 {"k1":"v1","k2":200} [200] diff --git a/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out b/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out index c8bbe10e57d6c1..1f6e68ea1de8cd 100644 --- a/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out +++ b/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out @@ -6186,7 +6186,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31" 26 \N \N 27 {"k1":"v1","k2":200} "v1" @@ -6215,7 +6215,7 @@ false 14 [123,456] [456] 15 ["abc","def"] ["def"] 16 [null,true,false,100,6.18,"abc"] [true] -17 [{"k1":"v41","k2":400},1,"a",3.14] [1] +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N 27 {"k1":"v1","k2":200} [200] @@ -6244,7 +6244,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N 27 {"k1":"v1","k2":200} [200] diff --git a/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out index c519579c75a1aa..722cae3d2c03f7 100644 --- a/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out @@ -8187,7 +8187,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31" 26 \N \N 27 {"k1":"v1","k2":200} "v1" @@ -8216,7 +8216,7 @@ false 14 [123,456] [456] 15 ["abc","def"] ["def"] 16 [null,true,false,100,6.18,"abc"] [true] -17 [{"k1":"v41","k2":400},1,"a",3.14] [1] +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N 27 {"k1":"v1","k2":200} [200] @@ -8245,7 +8245,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] [[400]] 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N 27 {"k1":"v1","k2":200} [200] diff --git a/regression-test/suites/auth_up_down_p0/load.groovy b/regression-test/suites/auth_up_down_p0/load.groovy index 84deb2cde2e206..a3f82513cc4374 100644 --- a/regression-test/suites/auth_up_down_p0/load.groovy +++ b/regression-test/suites/auth_up_down_p0/load.groovy @@ -185,6 +185,6 @@ suite("test_upgrade_downgrade_prepare_auth","p0,auth,restart_fe") { sql """GRANT USAGE_PRIV ON RESOURCE ${rg1} TO ${user1};""" connect(user1, "${pwd}", context.config.jdbcUrl) { def res = sql """SHOW RESOURCES;""" - assertTrue(res.size == 10) + assertTrue(res.size() == 10) } } diff --git a/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy b/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy index c2f9e4d74f4cbc..f1bedf23e2e45f 100644 --- a/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy +++ b/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy @@ -70,6 +70,6 @@ suite("test_upgrade_downgrade_compatibility_auth","p0,auth,restart_fe") { // resource group connect(user1, "${pwd}", context.config.jdbcUrl) { def res = sql """SHOW RESOURCES;""" - assertTrue(res.size == 10) + assertTrue(res.size() == 10) } } diff --git a/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy index 61b567e1d833a9..a772c8d0cc2c67 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy @@ -77,6 +77,7 @@ suite("test_iceberg_hadoop_catalog_kerberos", "p0,external,kerberos,external_doc """ def table = sql """ show tables like '%${test_tbl_name}' """ assert table.size() == 1 + sql """ insert into ${test_tbl_name} values ( '2024-05-26 12:34:56', diff --git a/regression-test/suites/external_table_p0/tvf/test_backends_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_backends_tvf.groovy index b06d0868eae3d3..241e15d6f6a378 100644 --- a/regression-test/suites/external_table_p0/tvf/test_backends_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_backends_tvf.groovy @@ -19,30 +19,30 @@ suite("test_backends_tvf","p0,external,tvf,external_docker") { List> table = sql """ select * from backends(); """ assertTrue(table.size() > 0) - assertEquals(25, table[0].size) + assertEquals(25, table[0].size()) // filter columns table = sql """ select BackendId, Host, Alive, TotalCapacity, Version, NodeRole from backends();""" assertTrue(table.size() > 0) - assertTrue(table[0].size == 6) + assertTrue(table[0].size() == 6) assertEquals(true, table[0][2]) // case insensitive table = sql """ select backendid, Host, alive, Totalcapacity, version, nodeRole from backends();""" assertTrue(table.size() > 0) - assertTrue(table[0].size == 6) + assertTrue(table[0].size() == 6) assertEquals(true, table[0][2]) // test aliase columns table = sql """ select backendid as id, Host as name, alive, NodeRole as r from backends();""" assertTrue(table.size() > 0) - assertTrue(table[0].size == 4) + assertTrue(table[0].size() == 4) assertEquals(true, table[0][2]) // test changing position of columns table = sql """ select Host as name, NodeRole as r, alive from backends();""" assertTrue(table.size() > 0) - assertTrue(table[0].size == 3) + assertTrue(table[0].size() == 3) assertEquals(true, table[0][2]) def res = sql """ select count(*) from backends() where alive = 1; """ diff --git a/regression-test/suites/external_table_p0/tvf/test_frontends_disks_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_frontends_disks_tvf.groovy index b4355d8dbafc18..509fa5a75dc3a3 100644 --- a/regression-test/suites/external_table_p0/tvf/test_frontends_disks_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_frontends_disks_tvf.groovy @@ -19,23 +19,23 @@ suite("test_frontends_disks_tvf", "p0,external,external_docker") { List> table = sql """ select * from `frontends_disks`(); """ assertTrue(table.size() > 0) - assertTrue(table[0].size == 10) + assertTrue(table[0].size() == 10) // filter columns table = sql """ select Name from `frontends_disks`();""" assertTrue(table.size() > 0) - assertTrue(table[0].size == 1) + assertTrue(table[0].size() == 1) // case insensitive table = sql """ select name, host, dirtype, dir from frontends_disks() order by dirtype;""" assertTrue(table.size() > 0) - assertTrue(table[0].size == 4) + assertTrue(table[0].size() == 4) assertEquals("audit-log", table[0][2]) // test aliase columns table = sql """ select name as n, host as h, dirtype as a from frontends_disks() order by dirtype; """ assertTrue(table.size() > 0) - assertTrue(table[0].size == 3) + assertTrue(table[0].size() == 3) assertEquals("audit-log", table[0][2]) // test changing position of columns diff --git a/regression-test/suites/external_table_p0/tvf/test_frontends_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_frontends_tvf.groovy index 0f0e4450cefea3..1132f5951f8063 100644 --- a/regression-test/suites/external_table_p0/tvf/test_frontends_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_frontends_tvf.groovy @@ -19,23 +19,23 @@ suite("test_frontends_tvf","p0,external,tvf,external_docker") { List> table = sql """ select * from `frontends`(); """ assertTrue(table.size() > 0) - assertTrue(table[0].size == 19) + assertTrue(table[0].size() == 19) // filter columns table = sql """ select Name from `frontends`();""" assertTrue(table.size() > 0) - assertTrue(table[0].size == 1) + assertTrue(table[0].size() == 1) // case insensitive table = sql """ select name, host, editlogport, httpport, alive from frontends();""" assertTrue(table.size() > 0) - assertTrue(table[0].size == 5) + assertTrue(table[0].size() == 5) assertEquals("true", table[0][4]) // test aliase columns table = sql """ select name as n, host as h, alive as a, editlogport as e from frontends(); """ assertTrue(table.size() > 0) - assertTrue(table[0].size == 4) + assertTrue(table[0].size() == 4) assertEquals("true", table[0][2]) // test changing position of columns diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load.groovy index 60b0710116d181..13fabe65ec0a06 100644 --- a/regression-test/suites/load_p0/stream_load/test_stream_load.groovy +++ b/regression-test/suites/load_p0/stream_load/test_stream_load.groovy @@ -1617,7 +1617,7 @@ suite("test_stream_load", "p0") { log.info(sql_result[0][0].toString()) log.info(sql_result[0][1].toString()) - log.info(sql_result[0].size.toString()) + log.info(sql_result[0].size().toString()) def beHost=sql_result[0][0] def beHttpPort=sql_result[0][1] diff --git a/regression-test/suites/nereids_syntax_p0/information_schema.groovy b/regression-test/suites/nereids_syntax_p0/information_schema.groovy index a86146b559e24d..65586c5a5d3c82 100644 --- a/regression-test/suites/nereids_syntax_p0/information_schema.groovy +++ b/regression-test/suites/nereids_syntax_p0/information_schema.groovy @@ -18,7 +18,7 @@ suite("information_schema") { List> table = sql """ select * from backends(); """ assertTrue(table.size() > 0) - assertTrue(table[0].size == 25) + assertTrue(table[0].size() == 25) sql "SELECT DATABASE();" sql "select USER();"