From 997b59f40bdb42ab4757bcba391e5a50ea2dadcc Mon Sep 17 00:00:00 2001 From: amorynan Date: Wed, 25 Dec 2024 10:58:06 +0800 Subject: [PATCH 1/3] fix core for new_json_reader --- be/src/vec/exec/format/json/new_json_reader.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index d79e86520741cd..71aad0ba00e197 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -1657,9 +1657,7 @@ Status NewJsonReader::_simdjson_write_data_to_column(simdjson::ondemand::value& data_serde = serde->get_nested_serdes()[0]; // kNullType will put 1 into the Null map, so there is no need to push 0 for kNullType. - if (value.type() != simdjson::ondemand::json_type::null) { - nullable_column->get_null_map_data().push_back(0); - } else { + if (value.type() == simdjson::ondemand::json_type::null) { nullable_column->insert_default(); *valid = true; return Status::OK(); @@ -1817,6 +1815,10 @@ Status NewJsonReader::_simdjson_write_data_to_column(simdjson::ondemand::value& } else { return Status::InternalError("Not support load to complex column."); } + // we should set nullmap at last to avoid column_nullable nullmap and data column size not same + if (nullable_column && value.type() != simdjson::ondemand::json_type::null) { + nullable_column->get_null_map_data().push_back(0); + } *valid = true; return Status::OK(); } From 2d330f960282d37d961f07d6c4757c4fcc98156d Mon Sep 17 00:00:00 2001 From: amorynan Date: Fri, 27 Dec 2024 15:54:21 +0800 Subject: [PATCH 2/3] add case --- .../test_error_json.json | 1 + .../load_error_json.groovy | 53 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 regression-test/data/load_p0/stream_load/load_json_parse_failed/test_error_json.json create mode 100644 regression-test/suites/load_p0/stream_load/load_json_parse_failed/load_error_json.groovy diff --git a/regression-test/data/load_p0/stream_load/load_json_parse_failed/test_error_json.json b/regression-test/data/load_p0/stream_load/load_json_parse_failed/test_error_json.json new file mode 100644 index 00000000000000..1578f2b4579eb0 --- /dev/null +++ b/regression-test/data/load_p0/stream_load/load_json_parse_failed/test_error_json.json @@ -0,0 +1 @@ +[{"id":3,"time":{"2024-06-27 22:52:40.41845+08:00"}}] diff --git a/regression-test/suites/load_p0/stream_load/load_json_parse_failed/load_error_json.groovy b/regression-test/suites/load_p0/stream_load/load_json_parse_failed/load_error_json.groovy new file mode 100644 index 00000000000000..b57ebc56e1f2b7 --- /dev/null +++ b/regression-test/suites/load_p0/stream_load/load_json_parse_failed/load_error_json.groovy @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("load_error_json") { + + sql """ DROP TABLE IF EXISTS test_error_json; """ + + sql """ + CREATE TABLE `test_error_json` ( + id INT, + time STRUCT + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`id`) BUCKETS AUTO + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + // load should return fail not core + streamLoad { + table "test_error_json" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strip_outer_array', 'true' + file 'test_error_json.json' + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("fail", json.Status.toLowerCase()) + } + } +} + From b0c489415647eeb0c07a2f88d1b09b18fd7d6bf0 Mon Sep 17 00:00:00 2001 From: amorynan Date: Mon, 30 Dec 2024 14:48:44 +0800 Subject: [PATCH 3/3] fix comment --- be/src/vec/exec/format/json/new_json_reader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index 736492e0ca4bc9..8b3802b0af0645 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -1816,7 +1816,7 @@ Status NewJsonReader::_simdjson_write_data_to_column(simdjson::ondemand::value& } else { return Status::InternalError("Not support load to complex column."); } - // we should set nullmap at last to avoid column_nullable nullmap and data column size not same + //We need to finally set the nullmap of column_nullable to keep the size consistent with data_column if (nullable_column && value.type() != simdjson::ondemand::json_type::null) { nullable_column->get_null_map_data().push_back(0); }