diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index a125654eadaecd..5820174e7fa061 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -399,20 +399,6 @@ Status NewJsonReader::_get_range_params() { if (_range.table_format_params.table_format_type == "hive") { _is_hive_table = true; } - if (_params.file_attributes.__isset.openx_json_ignore_malformed) { - _openx_json_ignore_malformed = _params.file_attributes.openx_json_ignore_malformed; - } - return Status::OK(); -} - -static Status ignore_malformed_json_append_null(Block& block) { - for (auto& column : block.get_columns()) { - if (!column->is_nullable()) [[unlikely]] { - return Status::DataQualityError("malformed json, but the column `{}` is not nullable.", - column->get_name()); - } - static_cast(column->assume_mutable().get())->insert_default(); - } return Status::OK(); } @@ -500,13 +486,8 @@ Status NewJsonReader::_vhandle_simple_json(RuntimeState* /*state*/, Block& block bool valid = false; if (_next_row >= _total_rows) { // parse json and generic document Status st = _parse_json(is_empty_row, eof); - if (st.is()) { - if (_is_load) { - continue; // continue to read next (for load, after this , already append error to file.) - } else if (_openx_json_ignore_malformed) { - RETURN_IF_ERROR(ignore_malformed_json_append_null(block)); - continue; - } + if (_is_load && st.is()) { + continue; // continue to read next (for load, after this , already append error to file.) } RETURN_IF_ERROR(st); if (*is_empty_row) { @@ -1315,15 +1296,9 @@ Status NewJsonReader::_simdjson_handle_simple_json(RuntimeState* /*state*/, Bloc // step2: get json value by json doc Status st = _get_json_value(&size, eof, &error, is_empty_row); - if (st.is()) { - if (_is_load) { - return Status::OK(); - } else if (_openx_json_ignore_malformed) { - RETURN_IF_ERROR(ignore_malformed_json_append_null(block)); - return Status::OK(); - } + if (_is_load && st.is()) { + return Status::OK(); } - RETURN_IF_ERROR(st); if (*is_empty_row || *eof) { return Status::OK(); diff --git a/be/src/vec/exec/format/json/new_json_reader.h b/be/src/vec/exec/format/json/new_json_reader.h index 6b42ca23b4f640..430f8c7af18713 100644 --- a/be/src/vec/exec/format/json/new_json_reader.h +++ b/be/src/vec/exec/format/json/new_json_reader.h @@ -293,22 +293,18 @@ class NewJsonReader : public GenericReader { int32_t skip_bitmap_col_idx {-1}; + bool _is_load = true; //Used to indicate whether it is a stream load. When loading, only data will be inserted into columnString. //If an illegal value is encountered during the load process, `_append_error_msg` should be called //instead of directly returning `Status::DataQualityError` - bool _is_load = true; + bool _is_hive_table = false; // In hive : create table xxx ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'; // Hive will not allow you to create columns with the same name but different case, including field names inside // structs, and will automatically convert uppercase names in create sql to lowercase.However, when Hive loads data // to table, the column names in the data may be uppercase,and there may be multiple columns with // the same name but different capitalization.We refer to the behavior of hive, convert all column names // in the data to lowercase,and use the last one as the insertion value - bool _is_hive_table = false; - - // hive : org.openx.data.jsonserde.JsonSerDe, `ignore.malformed.json` prop. - // If the variable is true, `null` will be inserted for llegal json format instead of return error. - bool _openx_json_ignore_malformed = false; DataTypeSerDeSPtrs _serdes; vectorized::DataTypeSerDe::FormatOptions _serde_options; diff --git a/docker/thirdparties/docker-compose/hive/scripts/auxlib/json-serde-1.3.9.tar.gz b/docker/thirdparties/docker-compose/hive/scripts/auxlib/json-serde-1.3.9.tar.gz deleted file mode 100644 index 1eb63aa772769c..00000000000000 Binary files a/docker/thirdparties/docker-compose/hive/scripts/auxlib/json-serde-1.3.9.tar.gz and /dev/null differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run76.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run76.hql deleted file mode 100644 index 1f83f93244564d..00000000000000 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run76.hql +++ /dev/null @@ -1,56 +0,0 @@ -create database if not exists openx_json; -use openx_json; - - -CREATE TABLE IF NOT EXISTS json_table ( - id INT, - name STRING, - numbers ARRAY, - scores MAP, - details STRUCT -) -ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' -LOCATION '/user/doris/preinstalled_data/json/openx_json/json_table'; - - -CREATE TABLE IF NOT EXISTS json_table_ignore_malformed ( - id INT, - name STRING, - numbers ARRAY, - scores MAP, - details STRUCT -) -ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' -WITH SERDEPROPERTIES ("ignore.malformed.json" = "true" ) -LOCATION '/user/doris/preinstalled_data/json/openx_json/json_table'; - - -CREATE TABLE json_data_arrays_tb ( - name string, age int) -ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' -LOCATION '/user/doris/preinstalled_data/json/openx_json/json_data_arrays_tb'; - - -CREATE TABLE IF NOT EXISTS scalar_to_array_tb( - id INT, - name STRING, - tags ARRAY -)ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' -LOCATION '/user/doris/preinstalled_data/json/openx_json/scalar_to_array_tb'; - - -CREATE TABLE IF NOT EXISTS json_one_column_table ( - name STRING, - id INT, - numbers ARRAY, - scores MAP, - details STRUCT -) -ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' -LOCATION '/user/doris/preinstalled_data/json/openx_json/json_one_column_table'; - -msck repair table json_table; -msck repair table json_table_ignore_malformed; -msck repair table json_data_arrays_tb; -msck repair table scalar_to_array_tb; -msck repair table json_one_column_table; \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh index 04ddf42c70c5b4..ac4c9ae448014f 100755 --- a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh +++ b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh @@ -20,17 +20,6 @@ set -e -x parallel=$(getconf _NPROCESSORS_ONLN) - - -AUX_LIB="/mnt/scripts/auxlib" -for file in "${AUX_LIB}"/*.tar.gz; do - [ -e "$file" ] || continue - tar -xzvf "$file" -C "$AUX_LIB" - echo "file = ${file}" -done -ls "${AUX_LIB}/" -mv "${AUX_LIB}"/ /opt/hive - nohup /opt/hive/bin/hive --service metastore & # wait lockfile diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_data_arrays_tb/1 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_data_arrays_tb/1 deleted file mode 100644 index 098bb346b503a5..00000000000000 --- a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_data_arrays_tb/1 +++ /dev/null @@ -1,2 +0,0 @@ -["John", 26 ] -["Mary", 23 ] \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_one_column_table/1 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_one_column_table/1 deleted file mode 100644 index d396f66a079168..00000000000000 --- a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_one_column_table/1 +++ /dev/null @@ -1,5 +0,0 @@ - -{"name":"bad1","id":5,"numbers":[1,2,3] -[1,2,3] -"just a string" -{"name":"bad2","id":6,"numbers":"not an array","scores":{"key4":40},"details":{"a":4,"b":"text","c":4000000}} diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_table/1 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_table/1 deleted file mode 100644 index 11a3edf6e8030d..00000000000000 --- a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_table/1 +++ /dev/null @@ -1,2 +0,0 @@ -{"id": 1, "name": "Alice", "numbers": [1, 2, 3], "scores": {"math": 90, "english": 85}, "details": {"a": 100, "b": "test1", "c": 1234567890}} -{"id": 2, "name": "Bob", "numbers": [4, 5], "scores": {"math": 80, "science": 95}, "details": {"a": 200, "b": "test2", "c": 9876543210}} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_table/2 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_table/2 deleted file mode 100644 index e77c1f49d858eb..00000000000000 --- a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/json_table/2 +++ /dev/null @@ -1,11 +0,0 @@ -{"id" 3 "name" "Bob", -"numbers": [ - 4 5 -], -"scores": { -"math": 80 -}, -"details" -: { -"a": 200 , "b" } "test2", "c": 9876543210 -}} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/scalar_to_array_tb/1 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/scalar_to_array_tb/1 deleted file mode 100644 index 24a9acc63a36a6..00000000000000 --- a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/openx_json/scalar_to_array_tb/1 +++ /dev/null @@ -1 +0,0 @@ -{"name":"Charlie","id":4,"tags":"flink"} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 33c945272e7143..dfd7d6fe4b8c0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -1084,14 +1084,6 @@ public boolean isPartitionedTable() { public void beforeMTMVRefresh(MTMV mtmv) throws DdlException { } - public boolean firstColumnIsString() { - List columns = getColumns(); - if (columns == null || columns.isEmpty()) { - return false; - } - return columns.get(0).getType().isScalarType(PrimitiveType.STRING); - } - public HoodieTableMetaClient getHudiClient() { return Env.getCurrentEnv() .getExtMetaCacheMgr() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java index d98bf8227e1e2f..9bb09225607674 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java @@ -95,7 +95,6 @@ public class HiveMetaStoreClientHelper { public static final String HIVE_JSON_SERDE = "org.apache.hive.hcatalog.data.JsonSerDe"; public static final String LEGACY_HIVE_JSON_SERDE = "org.apache.hadoop.hive.serde2.JsonSerDe"; - public static final String OPENX_JSON_SERDE = "org.openx.data.jsonserde.JsonSerDe"; public enum HiveFileFormat { TEXT_FILE(0, "text"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java index b06ea772fab3fe..74f3dcc1a9d0e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java @@ -55,11 +55,6 @@ public class HiveProperties { public static final String PROP_ESCAPE_CHAR = OpenCSVSerde.ESCAPECHAR; public static final String DEFAULT_ESCAPE_CHAR = "\\"; - // org.openx.data.jsonserde.JsonSerDe - public static final String PROP_OPENX_IGNORE_MALFORMED_JSON = "ignore.malformed.json"; - public static final String DEFAULT_OPENX_IGNORE_MALFORMED_JSON = "false"; - - public static final Set HIVE_SERDE_PROPERTIES = ImmutableSet.of( PROP_FIELD_DELIMITER, PROP_COLLECTION_DELIMITER_HIVE2, @@ -136,13 +131,6 @@ public static String getEscapeChar(Table table) { return HiveMetaStoreClientHelper.firstPresentOrDefault(DEFAULT_ESCAPE_CHAR, escapeChar); } - public static String getOpenxJsonIgnoreMalformed(Table table) { - Optional escapeChar = HiveMetaStoreClientHelper.getSerdeProperty(table, - PROP_OPENX_IGNORE_MALFORMED_JSON); - return HiveMetaStoreClientHelper.firstPresentOrDefault(DEFAULT_OPENX_IGNORE_MALFORMED_JSON, escapeChar); - } - - // Set properties to table public static void setTableProperties(Table table, Map properties) { HashMap serdeProps = new HashMap<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index c25037895803d5..890f6147f333a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -125,7 +125,7 @@ protected void doInitialize() throws UserException { this.hiveTransaction = new HiveTransaction(DebugUtil.printId(ConnectContext.get().queryId()), ConnectContext.get().getQualifiedUser(), hmsTable, hmsTable.isFullAcidTable()); Env.getCurrentHiveTransactionMgr().register(hiveTransaction); - skipCheckingAcidVersionFile = sessionVariable.skipCheckingAcidVersionFile; + skipCheckingAcidVersionFile = ConnectContext.get().getSessionVariable().skipCheckingAcidVersionFile; } } @@ -413,17 +413,6 @@ public TFileFormatType getFileFormatType() throws UserException { if (serDeLib.equals(HiveMetaStoreClientHelper.HIVE_JSON_SERDE) || serDeLib.equals(HiveMetaStoreClientHelper.LEGACY_HIVE_JSON_SERDE)) { type = TFileFormatType.FORMAT_JSON; - } else if (serDeLib.equals(HiveMetaStoreClientHelper.OPENX_JSON_SERDE)) { - if (!sessionVariable.isReadHiveJsonInOneColumn()) { - type = TFileFormatType.FORMAT_JSON; - } else if (sessionVariable.isReadHiveJsonInOneColumn() - && hmsTable.firstColumnIsString()) { - type = TFileFormatType.FORMAT_CSV_PLAIN; - } else { - throw new UserException("You set read_hive_json_in_one_column = true, but the first column of " - + "table " + hmsTable.getName() - + " is not a string column."); - } } else { type = TFileFormatType.FORMAT_CSV_PLAIN; } @@ -460,7 +449,7 @@ protected TFileAttributes getFileAttributes() throws UserException { fileAttributes.setTextParams(textParams); fileAttributes.setHeaderType(""); fileAttributes.setEnableTextValidateUtf8( - sessionVariable.enableTextValidateUtf8); + ConnectContext.get().getSessionVariable().enableTextValidateUtf8); } else if (serDeLib.equals("org.apache.hadoop.hive.serde2.OpenCSVSerde")) { TFileTextScanRangeParams textParams = new TFileTextScanRangeParams(); // set set properties of OpenCSVSerde @@ -478,7 +467,7 @@ protected TFileAttributes getFileAttributes() throws UserException { fileAttributes.setTrimDoubleQuotes(true); } fileAttributes.setEnableTextValidateUtf8( - sessionVariable.enableTextValidateUtf8); + ConnectContext.get().getSessionVariable().enableTextValidateUtf8); } else if (serDeLib.equals("org.apache.hive.hcatalog.data.JsonSerDe")) { TFileTextScanRangeParams textParams = new TFileTextScanRangeParams(); textParams.setColumnSeparator("\t"); @@ -492,37 +481,6 @@ protected TFileAttributes getFileAttributes() throws UserException { fileAttributes.setReadJsonByLine(true); fileAttributes.setStripOuterArray(false); fileAttributes.setHeaderType(""); - } else if (serDeLib.equals("org.openx.data.jsonserde.JsonSerDe")) { - if (!sessionVariable.isReadHiveJsonInOneColumn()) { - TFileTextScanRangeParams textParams = new TFileTextScanRangeParams(); - textParams.setColumnSeparator("\t"); - textParams.setLineDelimiter("\n"); - fileAttributes.setTextParams(textParams); - - fileAttributes.setJsonpaths(""); - fileAttributes.setJsonRoot(""); - fileAttributes.setNumAsString(true); - fileAttributes.setFuzzyParse(false); - fileAttributes.setReadJsonByLine(true); - fileAttributes.setStripOuterArray(false); - fileAttributes.setHeaderType(""); - - fileAttributes.setOpenxJsonIgnoreMalformed( - Boolean.parseBoolean(HiveProperties.getOpenxJsonIgnoreMalformed(table))); - } else if (sessionVariable.isReadHiveJsonInOneColumn() - && hmsTable.firstColumnIsString()) { - TFileTextScanRangeParams textParams = new TFileTextScanRangeParams(); - textParams.setLineDelimiter("\n"); - textParams.setColumnSeparator("\n"); - //First, perform row splitting according to `\n`. When performing column splitting, - // since there is no `\n`, only one column of data will be generated. - fileAttributes.setTextParams(textParams); - fileAttributes.setHeaderType(""); - } else { - throw new UserException("You set read_hive_json_in_one_column = true, but the first column of table " - + hmsTable.getName() - + " is not a string column."); - } } else { throw new UserException( "unsupported hive table serde: " + serDeLib); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 48ac010e83001d..86c47de612fe63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -708,8 +708,6 @@ public class SessionVariable implements Serializable, Writable { "enable_cooldown_replica_affinity"; public static final String SKIP_CHECKING_ACID_VERSION_FILE = "skip_checking_acid_version_file"; - public static final String READ_HIVE_JSON_IN_ONE_COLUMN = "read_hive_json_in_one_column"; - /** * Inserting overwrite for auto partition table allows creating partition for * datas which cannot find partition to overwrite. @@ -1222,17 +1220,6 @@ public enum IgnoreSplitType { @VariableMgr.VarAttr(name = PARALLEL_PREPARE_THRESHOLD, fuzzy = true) public int parallelPrepareThreshold = 32; - @VariableMgr.VarAttr(name = READ_HIVE_JSON_IN_ONE_COLUMN, - description = {"在读取hive json的时候，由于存在一些不支持的json格式，我们默认会报错。为了让用户使用体验更好，" - + "当该变量为true的时候，将一整行json读取到第一列中，用户可以自行选择对一整行json进行处理，例如JSON_PARSE。" - + "需要表的第一列的数据类型为string.", - "When reading hive json, we will report an error by default because there are some unsupported " - + "json formats. In order to provide users with a better experience, when this variable is true," - + "a whole line of json is read into the first column. Users can choose to process a whole line" - + "of json, such as JSON_PARSE. The data type of the first column of the table needs to" - + "be string."}) - private boolean readHiveJsonInOneColumn = false; - @VariableMgr.VarAttr(name = ENABLE_COST_BASED_JOIN_REORDER) private boolean enableJoinReorderBasedCost = false; @@ -3784,14 +3771,6 @@ public void setKeepCarriageReturn(boolean keepCarriageReturn) { this.keepCarriageReturn = keepCarriageReturn; } - public boolean isReadHiveJsonInOneColumn() { - return readHiveJsonInOneColumn; - } - - public void setReadHiveJsonInOneColumn(boolean readHiveJsonInOneColumn) { - this.readHiveJsonInOneColumn = readHiveJsonInOneColumn; - } - public boolean isDropTableIfCtasFailed() { return dropTableIfCtasFailed; } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 3165a6ac764db4..407c77fc340404 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -271,9 +271,6 @@ struct TFileAttributes { 11: optional i32 skip_lines; //For text type file reading, whether to enable utf8 encoding check.(Catalog && TVF) 12: optional bool enable_text_validate_utf8 = true; - // org.openx.data.jsonserde.JsonSerDe - 13: optional bool openx_json_ignore_malformed = false; - // for cloud copy into 1001: optional bool ignore_csv_redundant_col; } diff --git a/regression-test/data/external_table_p0/hive/test_hive_openx_json.out b/regression-test/data/external_table_p0/hive/test_hive_openx_json.out deleted file mode 100644 index 6eadea56694c85..00000000000000 --- a/regression-test/data/external_table_p0/hive/test_hive_openx_json.out +++ /dev/null @@ -1,25 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !q1 -- -\N \N \N \N \N -\N \N \N \N \N -\N \N \N \N \N -\N \N \N \N \N -\N \N \N \N \N -\N \N \N \N \N -\N \N \N \N \N -\N \N \N \N \N -\N \N \N \N \N -\N \N \N \N \N -1 Alice [1, 2, 3] {"math":90, "english":85} {"a":100, "b":"test1", "c":1234567890} -2 Bob [4, 5] {"math":80, "science":95} {"a":200, "b":"test2", "c":9876543210} - --- !2 -- -["John", 26 ] \N -["Mary", 23 ] \N - --- !3 -- -"just a string" \N \N \N \N -[1,2,3] \N \N \N \N -{"name":"bad1","id":5,"numbers":[1,2,3] \N \N \N \N -{"name":"bad2","id":6,"numbers":"not an array","scores":{"key4":40},"details":{"a":4,"b":"text","c":4000000}} \N \N \N \N - diff --git a/regression-test/suites/external_table_p0/hive/test_hive_openx_json.groovy b/regression-test/suites/external_table_p0/hive/test_hive_openx_json.groovy deleted file mode 100644 index b9698809c4d6dc..00000000000000 --- a/regression-test/suites/external_table_p0/hive/test_hive_openx_json.groovy +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_hive_openx_json", "p0,external,hive,external_docker,external_docker_hive") { - - - String enabled = context.config.otherConfigs.get("enableHiveTest") - if (enabled == null || !enabled.equalsIgnoreCase("true")) { - logger.info("diable Hive test.") - return; - } - - for (String hivePrefix : ["hive3"]) { - try { - sql """set enable_fallback_to_original_planner=false""" - String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") - String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") - String catalog_name = "${hivePrefix}_test_hive_openx_json" - String broker_name = "hdfs" - - sql """drop catalog if exists ${catalog_name}""" - sql """create catalog if not exists ${catalog_name} properties ( - 'type'='hms', - 'hive.metastore.uris'='thrift://${externalEnvIp}:${hms_port}' - );""" - sql """use `${catalog_name}`.`openx_json`""" - - try { - sql """ select * from json_table """; - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("DATA_QUALITY_ERROR")) - } - - order_qt_q1 """ select * from json_table_ignore_malformed """ - - - try{ - sql """ select * from json_data_arrays_tb """; - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("DATA_QUALITY_ERROR")) - } - - - try{ - sql """ select * from scalar_to_array_tb """; - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("DATA_QUALITY_ERROR")) - } - - sql """ set read_hive_json_in_one_column = true; """ - - order_qt_2 """ select * from json_data_arrays_tb """ - order_qt_3 """ select * from json_one_column_table """ - - try{ - sql """ select * from scalar_to_array_tb """; - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("is not a string column.")) - } - - - sql """drop catalog if exists ${catalog_name}""" - } finally { - } - } -}