From afda0cd4e3f042a8f671fc9aa3c5988ae269bf5b Mon Sep 17 00:00:00 2001 From: BePPPower Date: Mon, 14 Apr 2025 15:34:03 +0800 Subject: [PATCH 1/7] add 1 --- .../AvroFileFormatConfigurator.java | 52 ++++ .../fileformat/CsvFileFormatConfigurator.java | 189 ++++++++++++ .../fileformat/FileFormatConfigurator.java | 112 ++++++++ .../JsonFileFormatConfigurator.java | 116 ++++++++ .../fileformat/OrcFileFormatConfigurator.java | 62 ++++ .../ParquetFileFormatConfigurator.java | 71 +++++ .../fileformat/WalFileFormatConfigurator.java | 52 ++++ .../AvroFileFormatConfiguratorTest.java | 44 +++ .../CsvFileFormatConfiguratorTest.java | 268 ++++++++++++++++++ .../FileFormatConfiguratorTest.java | 34 +++ .../JsonFileFormatConfiguratorTest.java | 200 +++++++++++++ .../OrcFileFormatConfiguratorTest.java | 46 +++ .../ParquetFileFormatConfiguratorTest.java | 48 ++++ .../WalFileFormatConfiguratorTest.java | 43 +++ 14 files changed, 1337 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatConfiguratorTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java new file mode 100644 index 00000000000000..c097ba255fedf0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; +import org.apache.doris.thrift.TFileAttributes; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TResultFileSinkOptions; + +import java.util.Map; + +public class AvroFileFormatConfigurator extends FileFormatConfigurator { + public AvroFileFormatConfigurator(TFileFormatType fileFormatType) { + super(fileFormatType); + } + + @Override + public void analyzeFileFormatProperties(Map formatProperties) + throws AnalysisException { + } + + @Override + public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { + return null; + } + + @Override + public TResultFileSinkOptions toTResultFileSinkOptions() { + return null; + } + + @Override + public TFileAttributes toTFileAttributes() { + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java new file mode 100644 index 00000000000000..51f173941570c5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.analysis.Separator; +import org.apache.doris.catalog.Column; +import org.apache.doris.common.util.FileFormatUtils; +import org.apache.doris.common.util.Util; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; +import org.apache.doris.thrift.TFileAttributes; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TResultFileSinkOptions; +import org.apache.doris.thrift.TTextSerdeType; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Map; + +public class CsvFileFormatConfigurator extends FileFormatConfigurator { + public static final Logger LOG = LogManager.getLogger(CsvFileFormatConfigurator.class); + + private String headerType = ""; + private TTextSerdeType textSerdeType = TTextSerdeType.JSON_TEXT_SERDE; + private String columnSeparator = CsvFileFormatProperties.DEFAULT_COLUMN_SEPARATOR; + private String lineDelimiter = CsvFileFormatProperties.DEFAULT_LINE_DELIMITER; + private boolean trimDoubleQuotes; + private int skipLines; + private byte enclose; + + // used by tvf + // User specified csv columns, it will override columns got from file + private final List csvSchema = Lists.newArrayList(); + + String defaultColumnSeparator = CsvFileFormatProperties.DEFAULT_COLUMN_SEPARATOR; + + public CsvFileFormatConfigurator(TFileFormatType fileFormatType) { + super(fileFormatType); + } + + public CsvFileFormatConfigurator(TFileFormatType fileFormatType, String defaultColumnSeparator, + TTextSerdeType textSerdeType) { + super(fileFormatType); + this.defaultColumnSeparator = defaultColumnSeparator; + this.textSerdeType = textSerdeType; + } + + public CsvFileFormatConfigurator(TFileFormatType fileFormatType, String headerType) { + super(fileFormatType); + this.headerType = headerType; + } + + + @Override + public void analyzeFileFormatProperties(Map formatProperties) + throws AnalysisException { + try { + // check properties specified by user -- formatProperties + columnSeparator = formatProperties.getOrDefault(CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, + defaultColumnSeparator); + if (Strings.isNullOrEmpty(columnSeparator)) { + throw new AnalysisException("column_separator can not be empty."); + } + columnSeparator = Separator.convertSeparator(columnSeparator); + + lineDelimiter = formatProperties.getOrDefault(CsvFileFormatProperties.PROP_LINE_DELIMITER, + CsvFileFormatProperties.DEFAULT_LINE_DELIMITER); + if (Strings.isNullOrEmpty(lineDelimiter)) { + throw new AnalysisException("line_delimiter can not be empty."); + } + lineDelimiter = Separator.convertSeparator(lineDelimiter); + + String enclosedString = formatProperties.getOrDefault(CsvFileFormatProperties.PROP_ENCLOSE, ""); + if (!Strings.isNullOrEmpty(enclosedString)) { + if (enclosedString.length() > 1) { + throw new AnalysisException("enclose should not be longer than one byte."); + } + enclose = (byte) enclosedString.charAt(0); + if (enclose == 0) { + throw new AnalysisException("enclose should not be byte [0]."); + } + } + + trimDoubleQuotes = Boolean.valueOf( + formatProperties.getOrDefault(CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "")).booleanValue(); + skipLines = Integer.valueOf( + formatProperties.getOrDefault(CsvFileFormatProperties.PROP_SKIP_LINES, "0")).intValue(); + + String compressTypeStr = formatProperties.getOrDefault(CsvFileFormatProperties.PROP_COMPRESS_TYPE, + "UNKNOWN"); + try { + compressionType = Util.getFileCompressType(compressTypeStr); + } catch (IllegalArgumentException e) { + throw new AnalysisException("Compress type : " + compressTypeStr + " is not supported."); + } + + FileFormatUtils.parseCsvSchema(csvSchema, formatProperties.getOrDefault( + CsvFileFormatProperties.PROP_CSV_SCHEMA, "")); + if (LOG.isDebugEnabled()) { + LOG.debug("get csv schema: {}", csvSchema); + } + } catch (org.apache.doris.common.AnalysisException e) { + throw new AnalysisException(e.getMessage()); + } + } + + @Override + public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { + return null; + } + + @Override + public TResultFileSinkOptions toTResultFileSinkOptions() { + return null; + } + + @Override + public TFileAttributes toTFileAttributes() { + return null; + } + + public String getHeaderType() { + return headerType; + } + + public TTextSerdeType getTextSerdeType() { + return textSerdeType; + } + + public String getColumnSeparator() { + return columnSeparator; + } + + public String getLineDelimiter() { + return lineDelimiter; + } + + public boolean isTrimDoubleQuotes() { + return trimDoubleQuotes; + } + + public int getSkipLines() { + return skipLines; + } + + public byte getEnclose() { + return enclose; + } + + public List getCsvSchema() { + return csvSchema; + } + + public static class CsvFileFormatProperties { + public static final String DEFAULT_COLUMN_SEPARATOR = "\t"; + public static final String DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR = "\001"; + public static final String DEFAULT_LINE_DELIMITER = "\n"; + + public static final String PROP_COLUMN_SEPARATOR = "column_separator"; + public static final String PROP_LINE_DELIMITER = "line_delimiter"; + + public static final String PROP_SKIP_LINES = "skip_lines"; + public static final String PROP_CSV_SCHEMA = "csv_schema"; + public static final String PROP_COMPRESS = "compress"; + public static final String PROP_COMPRESS_TYPE = "compress_type"; + public static final String PROP_TRIM_DOUBLE_QUOTES = "trim_double_quotes"; + + public static final String PROP_ENCLOSE = "enclose"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java new file mode 100644 index 00000000000000..4de679e9ac2d69 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.datasource.property.fileformat.CsvFileFormatConfigurator.CsvFileFormatProperties; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; +import org.apache.doris.thrift.TFileAttributes; +import org.apache.doris.thrift.TFileCompressType; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TResultFileSinkOptions; +import org.apache.doris.thrift.TTextSerdeType; + +import java.util.Map; + +public abstract class FileFormatConfigurator { + public static class FileFormatProperties { + public static final String PROP_FORMAT = "format"; + public static final String FORMAT_PARQUET = "parquet"; + public static final String FORMAT_CSV = "csv"; + public static final String FORMAT_CSV_WITH_NAMES = "csv_with_names"; + public static final String FORMAT_CSV_WITH_NAMES_AND_TYPES = "csv_with_names_and_types"; + public static final String FORMAT_HIVE_TEXT = "hive_text"; + public static final String FORMAT_ORC = "orc"; + public static final String FORMAT_JSON = "json"; + public static final String FORMAT_AVRO = "avro"; + public static final String FORMAT_WAL = "wal"; + public static final String FORMAT_ARROW = "arrow"; + } + + protected TFileFormatType fileFormatType; + + protected TFileCompressType compressionType; + + public FileFormatConfigurator(TFileFormatType fileFormatType) { + this.fileFormatType = fileFormatType; + } + + /** + * Analyze user properties + * @param formatProperties properties specified by user + * @return properties needed by Doris + * @throws AnalysisException + */ + public abstract void analyzeFileFormatProperties(Map formatProperties) + throws AnalysisException; + + public abstract PFetchTableSchemaRequest toPFetchTableSchemaRequest(); + + public abstract TResultFileSinkOptions toTResultFileSinkOptions(); + + public abstract TFileAttributes toTFileAttributes(); + + public static FileFormatConfigurator createFileFormatChecker(String formatString) { + switch (formatString) { + case FileFormatProperties.FORMAT_CSV: + return new CsvFileFormatConfigurator(TFileFormatType.FORMAT_CSV_PLAIN); + case FileFormatProperties.FORMAT_HIVE_TEXT: + return new CsvFileFormatConfigurator(TFileFormatType.FORMAT_CSV_PLAIN, + CsvFileFormatProperties.DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR, + TTextSerdeType.HIVE_TEXT_SERDE); + case FileFormatProperties.FORMAT_CSV_WITH_NAMES: + return new CsvFileFormatConfigurator(TFileFormatType.FORMAT_CSV_PLAIN, + FileFormatProperties.FORMAT_CSV_WITH_NAMES); + case FileFormatProperties.FORMAT_CSV_WITH_NAMES_AND_TYPES: + return new CsvFileFormatConfigurator(TFileFormatType.FORMAT_CSV_PLAIN, + FileFormatProperties.FORMAT_CSV_WITH_NAMES_AND_TYPES); + case FileFormatProperties.FORMAT_PARQUET: + return new ParquetFileFormatConfigurator(TFileFormatType.FORMAT_PARQUET); + case FileFormatProperties.FORMAT_ORC: + return new OrcFileFormatConfigurator(TFileFormatType.FORMAT_ORC); + case FileFormatProperties.FORMAT_JSON: + return new JsonFileFormatConfigurator(TFileFormatType.FORMAT_JSON); + case FileFormatProperties.FORMAT_AVRO: + return new AvroFileFormatConfigurator(TFileFormatType.FORMAT_AVRO); + case FileFormatProperties.FORMAT_WAL: + return new WalFileFormatConfigurator(TFileFormatType.FORMAT_WAL); + default: + throw new AnalysisException("format:" + formatString + " is not supported."); + } + } + + public static FileFormatConfigurator createFileFormatChecker(Map formatProperties) + throws AnalysisException { + String formatString = formatProperties.getOrDefault(FileFormatProperties.PROP_FORMAT, "") + .toLowerCase(); + return createFileFormatChecker(formatString); + } + + public TFileFormatType getFileFormatType() { + return fileFormatType; + } + + public TFileCompressType getCompressionType() { + return compressionType; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java new file mode 100644 index 00000000000000..5dfffdbfb2173e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.property.fileformat.CsvFileFormatConfigurator.CsvFileFormatProperties; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; +import org.apache.doris.thrift.TFileAttributes; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TResultFileSinkOptions; + +import java.util.Map; + +public class JsonFileFormatConfigurator extends FileFormatConfigurator { + // from ExternalFileTableValuedFunction: + private String jsonRoot = ""; + private String jsonPaths = ""; + private boolean stripOuterArray; + private boolean readJsonByLine; + private boolean numAsString; + private boolean fuzzyParse; + + + public JsonFileFormatConfigurator(TFileFormatType fileFormatType) { + super(fileFormatType); + } + + @Override + public void analyzeFileFormatProperties(Map formatProperties) + throws AnalysisException { + // 这几个json应该移到json checker中 + jsonRoot = formatProperties.getOrDefault(JsonFileFormatProperties.PROP_JSON_ROOT, ""); + jsonPaths = formatProperties.getOrDefault(JsonFileFormatProperties.PROP_JSON_PATHS, ""); + readJsonByLine = Boolean.valueOf( + formatProperties.getOrDefault(JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "")).booleanValue(); + stripOuterArray = Boolean.valueOf( + formatProperties.getOrDefault(JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "")).booleanValue(); + numAsString = Boolean.valueOf( + formatProperties.getOrDefault(JsonFileFormatProperties.PROP_NUM_AS_STRING, "")).booleanValue(); + fuzzyParse = Boolean.valueOf( + formatProperties.getOrDefault(JsonFileFormatProperties.PROP_FUZZY_PARSE, "")).booleanValue(); + + String compressTypeStr = formatProperties.getOrDefault(CsvFileFormatProperties.PROP_COMPRESS_TYPE, + "UNKNOWN"); + try { + compressionType = Util.getFileCompressType(compressTypeStr); + } catch (IllegalArgumentException e) { + throw new AnalysisException("Compress type : " + compressTypeStr + " is not supported."); + } + } + + @Override + public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { + return null; + } + + @Override + public TResultFileSinkOptions toTResultFileSinkOptions() { + return null; + } + + @Override + public TFileAttributes toTFileAttributes() { + return null; + } + + public String getJsonRoot() { + return jsonRoot; + } + + public String getJsonPaths() { + return jsonPaths; + } + + public boolean isStripOuterArray() { + return stripOuterArray; + } + + public boolean isReadJsonByLine() { + return readJsonByLine; + } + + public boolean isNumAsString() { + return numAsString; + } + + public boolean isFuzzyParse() { + return fuzzyParse; + } + + + public static class JsonFileFormatProperties { + public static final String PROP_JSON_ROOT = "json_root"; + public static final String PROP_JSON_PATHS = "jsonpaths"; + public static final String PROP_STRIP_OUTER_ARRAY = "strip_outer_array"; + public static final String PROP_READ_JSON_BY_LINE = "read_json_by_line"; + public static final String PROP_NUM_AS_STRING = "num_as_string"; + public static final String PROP_FUZZY_PARSE = "fuzzy_parse"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java new file mode 100644 index 00000000000000..f8351d386ca293 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; +import org.apache.doris.thrift.TFileAttributes; +import org.apache.doris.thrift.TFileCompressType; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TResultFileSinkOptions; + +import java.util.Map; + +public class OrcFileFormatConfigurator extends FileFormatConfigurator { + private TFileCompressType orcCompressionType = TFileCompressType.ZLIB; + + public OrcFileFormatConfigurator(TFileFormatType fileFormatType) { + super(fileFormatType); + } + + @Override + public void analyzeFileFormatProperties(Map formatProperties) + throws AnalysisException { + } + + @Override + public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { + return null; + } + + @Override + public TResultFileSinkOptions toTResultFileSinkOptions() { + return null; + } + + @Override + public TFileAttributes toTFileAttributes() { + return null; + } + + public TFileCompressType getOrcCompressionType() { + return orcCompressionType; + } + + public static class OrcFileFormatProperties { + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java new file mode 100644 index 00000000000000..f53dacc18f0aaf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; +import org.apache.doris.thrift.TFileAttributes; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TParquetCompressionType; +import org.apache.doris.thrift.TParquetVersion; +import org.apache.doris.thrift.TResultFileSinkOptions; + +import java.util.Map; + +public class ParquetFileFormatConfigurator extends FileFormatConfigurator { + private TParquetCompressionType parquetCompressionType = TParquetCompressionType.SNAPPY; + private boolean parquetDisableDictionary = false; + + public ParquetFileFormatConfigurator(TFileFormatType fileFormatType) { + super(fileFormatType); + } + + @Override + public void analyzeFileFormatProperties(Map formatProperties) + throws AnalysisException { + } + + @Override + public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { + return null; + } + + @Override + public TResultFileSinkOptions toTResultFileSinkOptions() { + return null; + } + + @Override + public TFileAttributes toTFileAttributes() { + return null; + } + + public TParquetCompressionType getParquetCompressionType() { + return parquetCompressionType; + } + + public boolean isParquetDisableDictionary() { + return parquetDisableDictionary; + } + + public static class ParquetFileFormatProperties { + public static final String PARQUET_DISABLE_DICTIONARY = "disable_dictionary"; + public static final TParquetVersion parquetVersion = TParquetVersion.PARQUET_1_0; + public static final String PARQUET_VERSION = "version"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java new file mode 100644 index 00000000000000..31979bfc3354aa --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; +import org.apache.doris.thrift.TFileAttributes; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TResultFileSinkOptions; + +import java.util.Map; + +public class WalFileFormatConfigurator extends FileFormatConfigurator { + public WalFileFormatConfigurator(TFileFormatType fileFormatType) { + super(fileFormatType); + } + + @Override + public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { + return null; + } + + @Override + public TResultFileSinkOptions toTResultFileSinkOptions() { + return null; + } + + @Override + public TFileAttributes toTFileAttributes() { + return null; + } + + @Override + public void analyzeFileFormatProperties(Map formatProperties) + throws AnalysisException { + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java new file mode 100644 index 00000000000000..208e842e0a55fa --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.thrift.TFileFormatType; + +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + + +public class AvroFileFormatConfiguratorTest { + + private AvroFileFormatConfigurator checker; + + @Before + public void setUp() { + checker = new AvroFileFormatConfigurator(TFileFormatType.FORMAT_AVRO); + } + + @Test + public void testAnalyzeFileFormatProperties() { + Map properties = new HashMap<>(); + // Add properties if needed + checker.analyzeFileFormatProperties(properties); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java new file mode 100644 index 00000000000000..c4c2b876887fe7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.thrift.TFileCompressType; +import org.apache.doris.thrift.TFileFormatType; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class CsvFileFormatConfiguratorTest { + + private CsvFileFormatConfigurator csvConfigurator; + + @Before + public void setUp() { + csvConfigurator = new CsvFileFormatConfigurator(TFileFormatType.FORMAT_CSV_PLAIN); + } + + @Test + public void testAnalyzeFileFormatPropertiesValid() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ","); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, "\n"); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "1"); + + csvConfigurator.analyzeFileFormatProperties(properties); + + Assert.assertEquals(",", csvConfigurator.getColumnSeparator()); + Assert.assertEquals("\n", csvConfigurator.getLineDelimiter()); + Assert.assertEquals(1, csvConfigurator.getSkipLines()); + } + + @Test + public void testAnalyzeFileFormatPropertiesInvalidSeparator() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ""); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesInvalidLineDelimiter() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, ""); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesInvalidEnclose() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, "invalid"); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidEnclose() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, "\""); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals((byte) '"', csvConfigurator.getEnclose()); + } + + @Test + public void testAnalyzeFileFormatPropertiesSkipLinesNegative() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "-1"); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesSkipLinesLargeValue() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "1000"); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(1000, csvConfigurator.getSkipLines()); + } + + @Test + public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesTrue() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(true, csvConfigurator.isTrimDoubleQuotes()); + } + + @Test + public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesFalse() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "false"); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(false, csvConfigurator.isTrimDoubleQuotes()); + } + + @Test + public void testAnalyzeFileFormatPropertiesInvalidCompressType() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COMPRESS_TYPE, "invalid"); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidCompressType() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COMPRESS_TYPE, "gzip"); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(TFileCompressType.GZ, csvConfigurator.getCompressionType()); + } + + @Test + public void testAnalyzeFileFormatPropertiesEmptyCsvSchema() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, ""); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidCsvSchema() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, "column1,column2,column3"); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidEncloseMultipleCharacters() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, "\"\""); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidEncloseEmpty() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, ""); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesSkipLinesAsString() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "abc"); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidColumnSeparator() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ";"); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(";", csvConfigurator.getColumnSeparator()); + } + + @Test + public void testAnalyzeFileFormatPropertiesLineDelimiterAsString() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, "abc"); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidLineDelimiter() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, "\r\n"); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals("\r\n", csvConfigurator.getLineDelimiter()); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidTrimDoubleQuotes() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(true, csvConfigurator.isTrimDoubleQuotes()); + } + + @Test + public void testAnalyzeFileFormatPropertiesInvalidTrimDoubleQuotes() { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "invalid"); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpaces() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, " column1 , column2 , column3 "); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); + Assert.assertEquals("column1", csvConfigurator.getCsvSchema().get(0).getName().trim()); + Assert.assertEquals("column2", csvConfigurator.getCsvSchema().get(1).getName().trim()); + Assert.assertEquals("column3", csvConfigurator.getCsvSchema().get(2).getName().trim()); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpecialCharacters() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, "col1@#$,col2&*(),col3"); + + csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); + Assert.assertEquals("col1@#$", csvConfigurator.getCsvSchema().get(0).getName()); + Assert.assertEquals("col2&*()", csvConfigurator.getCsvSchema().get(1).getName()); + Assert.assertEquals("col3", csvConfigurator.getCsvSchema().get(2).getName()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatConfiguratorTest.java new file mode 100644 index 00000000000000..d1f2f45092babb --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatConfiguratorTest.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.nereids.exceptions.AnalysisException; + +import org.junit.Assert; +import org.junit.Test; + + +public class FileFormatConfiguratorTest { + + @Test + public void testCreateFileFormatCheckerInvalidFormat() { + Assert.assertThrows(AnalysisException.class, () -> { + FileFormatConfigurator.createFileFormatChecker("invalid_format"); + }); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java new file mode 100644 index 00000000000000..40cb9f0f58dfff --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.thrift.TFileFormatType; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class JsonFileFormatConfiguratorTest { + + private JsonFileFormatConfigurator configurator; + + @Before + public void setUp() { + configurator = new JsonFileFormatConfigurator(TFileFormatType.FORMAT_JSON); + } + + @Test + public void testAnalyzeFileFormatPropertiesEmpty() throws AnalysisException { + Map properties = new HashMap<>(); + configurator.analyzeFileFormatProperties(properties); + + Assert.assertEquals("", configurator.getJsonRoot()); + Assert.assertEquals("", configurator.getJsonPaths()); + Assert.assertEquals(false, configurator.isStripOuterArray()); + Assert.assertEquals(false, configurator.isReadJsonByLine()); + Assert.assertEquals(false, configurator.isNumAsString()); + Assert.assertEquals(false, configurator.isFuzzyParse()); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidJsonRoot() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_ROOT, "data.items"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals("data.items", configurator.getJsonRoot()); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidJsonPaths() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, + "[\"$.name\", \"$.age\", \"$.city\"]"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals("[\"$.name\", \"$.age\", \"$.city\"]", configurator.getJsonPaths()); + } + + @Test + public void testAnalyzeFileFormatPropertiesStripOuterArrayTrue() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "true"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(true, configurator.isStripOuterArray()); + } + + @Test + public void testAnalyzeFileFormatPropertiesStripOuterArrayFalse() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "false"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(false, configurator.isStripOuterArray()); + } + + @Test + public void testAnalyzeFileFormatPropertiesReadJsonByLineTrue() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "true"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(true, configurator.isReadJsonByLine()); + } + + @Test + public void testAnalyzeFileFormatPropertiesReadJsonByLineFalse() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "false"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(false, configurator.isReadJsonByLine()); + } + + @Test + public void testAnalyzeFileFormatPropertiesNumAsStringTrue() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_NUM_AS_STRING, "true"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(true, configurator.isNumAsString()); + } + + @Test + public void testAnalyzeFileFormatPropertiesNumAsStringFalse() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_NUM_AS_STRING, "false"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(false, configurator.isNumAsString()); + } + + @Test + public void testAnalyzeFileFormatPropertiesFuzzyParseTrue() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "true"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(true, configurator.isFuzzyParse()); + } + + @Test + public void testAnalyzeFileFormatPropertiesFuzzyParseFalse() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(false, configurator.isFuzzyParse()); + } + + @Test + public void testAnalyzeFileFormatPropertiesInvalidBooleanValue() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "invalid"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(false, configurator.isFuzzyParse()); + } + + @Test + public void testAnalyzeFileFormatPropertiesAllProperties() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_ROOT, "data.records"); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, "[\"$.id\", \"$.name\"]"); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "true"); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "true"); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_NUM_AS_STRING, "true"); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "true"); + + configurator.analyzeFileFormatProperties(properties); + + Assert.assertEquals("data.records", configurator.getJsonRoot()); + Assert.assertEquals("[\"$.id\", \"$.name\"]", configurator.getJsonPaths()); + Assert.assertEquals(true, configurator.isStripOuterArray()); + Assert.assertEquals(true, configurator.isReadJsonByLine()); + Assert.assertEquals(true, configurator.isNumAsString()); + Assert.assertEquals(true, configurator.isFuzzyParse()); + } + + @Test + public void testAnalyzeFileFormatPropertiesSpecialCharactersInJsonRoot() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_ROOT, "data.special@#$%^&*()"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals("data.special@#$%^&*()", configurator.getJsonRoot()); + } + + @Test + public void testAnalyzeFileFormatPropertiesComplexJsonPaths() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, + "[\"$.deeply.nested[0].array[*].field\", \"$.complex.path[?(@.type=='value')]\"]"); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals("[\"$.deeply.nested[0].array[*].field\", \"$.complex.path[?(@.type=='value')]\"]", + configurator.getJsonPaths()); + } + + @Test + public void testAnalyzeFileFormatPropertiesEmptyJsonPaths() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, ""); + + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals("", configurator.getJsonPaths()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java new file mode 100644 index 00000000000000..1283864e6e4141 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.thrift.TFileCompressType; +import org.apache.doris.thrift.TFileFormatType; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class OrcFileFormatConfiguratorTest { + + private OrcFileFormatConfigurator configurator; + + @Before + public void setUp() { + configurator = new OrcFileFormatConfigurator(TFileFormatType.FORMAT_ORC); + } + + @Test + public void testAnalyzeFileFormatProperties() { + Map properties = new HashMap<>(); + // Add properties if needed + configurator.analyzeFileFormatProperties(properties); + Assert.assertEquals(TFileCompressType.ZLIB, configurator.getOrcCompressionType()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java new file mode 100644 index 00000000000000..3b217ba61bb1f3 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TParquetCompressionType; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class ParquetFileFormatConfiguratorTest { + + private ParquetFileFormatConfigurator configurator; + + @Before + public void setUp() { + configurator = new ParquetFileFormatConfigurator(TFileFormatType.FORMAT_PARQUET); + } + + @Test + public void testAnalyzeFileFormatProperties() { + Map properties = new HashMap<>(); + // Add properties if needed + configurator.analyzeFileFormatProperties(properties); + + Assert.assertEquals(TParquetCompressionType.SNAPPY, configurator.getParquetCompressionType()); + Assert.assertEquals(false, configurator.isParquetDisableDictionary()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java new file mode 100644 index 00000000000000..25451122da0685 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.thrift.TFileFormatType; + +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class WalFileFormatConfiguratorTest { + + private WalFileFormatConfigurator configurator; + + @Before + public void setUp() { + configurator = new WalFileFormatConfigurator(TFileFormatType.FORMAT_WAL); + } + + @Test + public void testAnalyzeFileFormatProperties() { + Map properties = new HashMap<>(); + // Add properties if needed + configurator.analyzeFileFormatProperties(properties); + } +} From db0ace536185c784a332cefdef42c16298e0c315 Mon Sep 17 00:00:00 2001 From: BePPPower Date: Mon, 21 Apr 2025 16:51:40 +0800 Subject: [PATCH 2/7] fix 2 --- .../AvroFileFormatConfigurator.java | 8 +- .../fileformat/CsvFileFormatConfigurator.java | 52 ++++++--- .../fileformat/FileFormatConfigurator.java | 21 +++- .../JsonFileFormatConfigurator.java | 38 +++++-- .../fileformat/OrcFileFormatConfigurator.java | 8 +- .../ParquetFileFormatConfigurator.java | 8 +- .../fileformat/WalFileFormatConfigurator.java | 8 +- .../AvroFileFormatConfiguratorTest.java | 2 +- .../CsvFileFormatConfiguratorTest.java | 107 ++++++++++-------- .../JsonFileFormatConfiguratorTest.java | 32 +++--- .../OrcFileFormatConfiguratorTest.java | 2 +- .../ParquetFileFormatConfiguratorTest.java | 2 +- .../WalFileFormatConfiguratorTest.java | 2 +- 13 files changed, 187 insertions(+), 103 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java index c097ba255fedf0..e5ad5da0f48059 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java @@ -21,6 +21,7 @@ import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileTextScanRangeParams; import org.apache.doris.thrift.TResultFileSinkOptions; import java.util.Map; @@ -31,7 +32,7 @@ public AvroFileFormatConfigurator(TFileFormatType fileFormatType) { } @Override - public void analyzeFileFormatProperties(Map formatProperties) + public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException { } @@ -47,6 +48,9 @@ public TResultFileSinkOptions toTResultFileSinkOptions() { @Override public TFileAttributes toTFileAttributes() { - return null; + TFileAttributes fileAttributes = new TFileAttributes(); + TFileTextScanRangeParams fileTextScanRangeParams = new TFileTextScanRangeParams(); + fileAttributes.setTextParams(fileTextScanRangeParams); + return fileAttributes; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java index 51f173941570c5..f863bb813e5ce8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java @@ -23,8 +23,10 @@ import org.apache.doris.common.util.Util; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileTextScanRangeParams; import org.apache.doris.thrift.TResultFileSinkOptions; import org.apache.doris.thrift.TTextSerdeType; @@ -71,25 +73,26 @@ public CsvFileFormatConfigurator(TFileFormatType fileFormatType, String headerTy @Override - public void analyzeFileFormatProperties(Map formatProperties) + public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException { try { // check properties specified by user -- formatProperties - columnSeparator = formatProperties.getOrDefault(CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, - defaultColumnSeparator); + columnSeparator = getOrDefaultAndRemove(formatProperties, CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, + defaultColumnSeparator, isRemoveOriginProperty); if (Strings.isNullOrEmpty(columnSeparator)) { throw new AnalysisException("column_separator can not be empty."); } columnSeparator = Separator.convertSeparator(columnSeparator); - lineDelimiter = formatProperties.getOrDefault(CsvFileFormatProperties.PROP_LINE_DELIMITER, - CsvFileFormatProperties.DEFAULT_LINE_DELIMITER); + lineDelimiter = getOrDefaultAndRemove(formatProperties, CsvFileFormatProperties.PROP_LINE_DELIMITER, + CsvFileFormatProperties.DEFAULT_LINE_DELIMITER, isRemoveOriginProperty); if (Strings.isNullOrEmpty(lineDelimiter)) { throw new AnalysisException("line_delimiter can not be empty."); } lineDelimiter = Separator.convertSeparator(lineDelimiter); - String enclosedString = formatProperties.getOrDefault(CsvFileFormatProperties.PROP_ENCLOSE, ""); + String enclosedString = getOrDefaultAndRemove(formatProperties, CsvFileFormatProperties.PROP_ENCLOSE, + "", isRemoveOriginProperty); if (!Strings.isNullOrEmpty(enclosedString)) { if (enclosedString.length() > 1) { throw new AnalysisException("enclose should not be longer than one byte."); @@ -100,21 +103,25 @@ public void analyzeFileFormatProperties(Map formatProperties) } } - trimDoubleQuotes = Boolean.valueOf( - formatProperties.getOrDefault(CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "")).booleanValue(); - skipLines = Integer.valueOf( - formatProperties.getOrDefault(CsvFileFormatProperties.PROP_SKIP_LINES, "0")).intValue(); + trimDoubleQuotes = Boolean.valueOf(getOrDefaultAndRemove(formatProperties, + CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "", isRemoveOriginProperty)) + .booleanValue(); + skipLines = Integer.valueOf(getOrDefaultAndRemove(formatProperties, + CsvFileFormatProperties.PROP_SKIP_LINES, "0", isRemoveOriginProperty)).intValue(); + if (skipLines < 0) { + throw new AnalysisException("skipLines should not be less than 0."); + } - String compressTypeStr = formatProperties.getOrDefault(CsvFileFormatProperties.PROP_COMPRESS_TYPE, - "UNKNOWN"); + String compressTypeStr = getOrDefaultAndRemove(formatProperties, + CsvFileFormatProperties.PROP_COMPRESS_TYPE, "UNKNOWN", isRemoveOriginProperty); try { compressionType = Util.getFileCompressType(compressTypeStr); } catch (IllegalArgumentException e) { throw new AnalysisException("Compress type : " + compressTypeStr + " is not supported."); } - FileFormatUtils.parseCsvSchema(csvSchema, formatProperties.getOrDefault( - CsvFileFormatProperties.PROP_CSV_SCHEMA, "")); + FileFormatUtils.parseCsvSchema(csvSchema, getOrDefaultAndRemove(formatProperties, + CsvFileFormatProperties.PROP_CSV_SCHEMA, "", isRemoveOriginProperty)); if (LOG.isDebugEnabled()) { LOG.debug("get csv schema: {}", csvSchema); } @@ -133,9 +140,23 @@ public TResultFileSinkOptions toTResultFileSinkOptions() { return null; } + // The method `analyzeFileFormatProperties` must have been called once before this method @Override public TFileAttributes toTFileAttributes() { - return null; + TFileAttributes fileAttributes = new TFileAttributes(); + TFileTextScanRangeParams fileTextScanRangeParams = new TFileTextScanRangeParams(); + fileTextScanRangeParams.setColumnSeparator(this.columnSeparator); + fileTextScanRangeParams.setLineDelimiter(this.lineDelimiter); + if (this.enclose != 0) { + fileTextScanRangeParams.setEnclose(this.enclose); + } + fileAttributes.setTextParams(fileTextScanRangeParams); + fileAttributes.setHeaderType(headerType); + fileAttributes.setTrimDoubleQuotes(trimDoubleQuotes); + fileAttributes.setSkipLines(skipLines); + fileAttributes.setEnableTextValidateUtf8( + ConnectContext.get().getSessionVariable().enableTextValidateUtf8); + return fileAttributes; } public String getHeaderType() { @@ -180,7 +201,6 @@ public static class CsvFileFormatProperties { public static final String PROP_SKIP_LINES = "skip_lines"; public static final String PROP_CSV_SCHEMA = "csv_schema"; - public static final String PROP_COMPRESS = "compress"; public static final String PROP_COMPRESS_TYPE = "compress_type"; public static final String PROP_TRIM_DOUBLE_QUOTES = "trim_double_quotes"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java index 4de679e9ac2d69..cd4aa7f238da94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java @@ -51,13 +51,21 @@ public FileFormatConfigurator(TFileFormatType fileFormatType) { this.fileFormatType = fileFormatType; } + /** + * + * @param formatProperties + * @return properties needed by Doris + * @throws AnalysisException + */ + /** * Analyze user properties * @param formatProperties properties specified by user - * @return properties needed by Doris + * @param isRemoveOriginProperty if this param is set to true, then this method would remove the origin property * @throws AnalysisException */ - public abstract void analyzeFileFormatProperties(Map formatProperties) + public abstract void analyzeFileFormatProperties( + Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException; public abstract PFetchTableSchemaRequest toPFetchTableSchemaRequest(); @@ -102,6 +110,15 @@ public static FileFormatConfigurator createFileFormatChecker(Map return createFileFormatChecker(formatString); } + protected String getOrDefaultAndRemove(Map props, String key, String defaultValue, + boolean isRemove) { + String value = props.getOrDefault(key, defaultValue); + if (isRemove) { + props.remove(key); + } + return value; + } + public TFileFormatType getFileFormatType() { return fileFormatType; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java index 5dfffdbfb2173e..c05bf24ec3ba9e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java @@ -17,12 +17,13 @@ package org.apache.doris.datasource.property.fileformat; +import org.apache.doris.common.util.FileFormatConstants; import org.apache.doris.common.util.Util; -import org.apache.doris.datasource.property.fileformat.CsvFileFormatConfigurator.CsvFileFormatProperties; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileTextScanRangeParams; import org.apache.doris.thrift.TResultFileSinkOptions; import java.util.Map; @@ -42,22 +43,28 @@ public JsonFileFormatConfigurator(TFileFormatType fileFormatType) { } @Override - public void analyzeFileFormatProperties(Map formatProperties) + public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException { // 这几个json应该移到json checker中 - jsonRoot = formatProperties.getOrDefault(JsonFileFormatProperties.PROP_JSON_ROOT, ""); - jsonPaths = formatProperties.getOrDefault(JsonFileFormatProperties.PROP_JSON_PATHS, ""); + jsonRoot = getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_JSON_ROOT, + "", isRemoveOriginProperty); + jsonPaths = getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_JSON_PATHS, + "", isRemoveOriginProperty); readJsonByLine = Boolean.valueOf( - formatProperties.getOrDefault(JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "")).booleanValue(); + getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_READ_JSON_BY_LINE, + "", isRemoveOriginProperty)).booleanValue(); stripOuterArray = Boolean.valueOf( - formatProperties.getOrDefault(JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "")).booleanValue(); + getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_STRIP_OUTER_ARRAY, + "", isRemoveOriginProperty)).booleanValue(); numAsString = Boolean.valueOf( - formatProperties.getOrDefault(JsonFileFormatProperties.PROP_NUM_AS_STRING, "")).booleanValue(); + getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_NUM_AS_STRING, + "", isRemoveOriginProperty)).booleanValue(); fuzzyParse = Boolean.valueOf( - formatProperties.getOrDefault(JsonFileFormatProperties.PROP_FUZZY_PARSE, "")).booleanValue(); + getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_FUZZY_PARSE, + "", isRemoveOriginProperty)).booleanValue(); - String compressTypeStr = formatProperties.getOrDefault(CsvFileFormatProperties.PROP_COMPRESS_TYPE, - "UNKNOWN"); + String compressTypeStr = getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_COMPRESS_TYPE, + "UNKNOWN", isRemoveOriginProperty); try { compressionType = Util.getFileCompressType(compressTypeStr); } catch (IllegalArgumentException e) { @@ -77,7 +84,16 @@ public TResultFileSinkOptions toTResultFileSinkOptions() { @Override public TFileAttributes toTFileAttributes() { - return null; + TFileAttributes fileAttributes = new TFileAttributes(); + TFileTextScanRangeParams fileTextScanRangeParams = new TFileTextScanRangeParams(); + fileAttributes.setTextParams(fileTextScanRangeParams); + fileAttributes.setJsonRoot(jsonRoot); + fileAttributes.setJsonpaths(jsonPaths); + fileAttributes.setReadJsonByLine(readJsonByLine); + fileAttributes.setStripOuterArray(stripOuterArray); + fileAttributes.setNumAsString(numAsString); + fileAttributes.setFuzzyParse(fuzzyParse); + return fileAttributes; } public String getJsonRoot() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java index f8351d386ca293..084c5a81d073fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java @@ -22,6 +22,7 @@ import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileTextScanRangeParams; import org.apache.doris.thrift.TResultFileSinkOptions; import java.util.Map; @@ -34,7 +35,7 @@ public OrcFileFormatConfigurator(TFileFormatType fileFormatType) { } @Override - public void analyzeFileFormatProperties(Map formatProperties) + public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException { } @@ -50,7 +51,10 @@ public TResultFileSinkOptions toTResultFileSinkOptions() { @Override public TFileAttributes toTFileAttributes() { - return null; + TFileAttributes fileAttributes = new TFileAttributes(); + TFileTextScanRangeParams fileTextScanRangeParams = new TFileTextScanRangeParams(); + fileAttributes.setTextParams(fileTextScanRangeParams); + return fileAttributes; } public TFileCompressType getOrcCompressionType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java index f53dacc18f0aaf..6e22c789501dc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java @@ -21,6 +21,7 @@ import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileTextScanRangeParams; import org.apache.doris.thrift.TParquetCompressionType; import org.apache.doris.thrift.TParquetVersion; import org.apache.doris.thrift.TResultFileSinkOptions; @@ -36,7 +37,7 @@ public ParquetFileFormatConfigurator(TFileFormatType fileFormatType) { } @Override - public void analyzeFileFormatProperties(Map formatProperties) + public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException { } @@ -52,7 +53,10 @@ public TResultFileSinkOptions toTResultFileSinkOptions() { @Override public TFileAttributes toTFileAttributes() { - return null; + TFileAttributes fileAttributes = new TFileAttributes(); + TFileTextScanRangeParams fileTextScanRangeParams = new TFileTextScanRangeParams(); + fileAttributes.setTextParams(fileTextScanRangeParams); + return fileAttributes; } public TParquetCompressionType getParquetCompressionType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java index 31979bfc3354aa..7b64a2e9eb2c4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java @@ -21,6 +21,7 @@ import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileTextScanRangeParams; import org.apache.doris.thrift.TResultFileSinkOptions; import java.util.Map; @@ -42,11 +43,14 @@ public TResultFileSinkOptions toTResultFileSinkOptions() { @Override public TFileAttributes toTFileAttributes() { - return null; + TFileAttributes fileAttributes = new TFileAttributes(); + TFileTextScanRangeParams fileTextScanRangeParams = new TFileTextScanRangeParams(); + fileAttributes.setTextParams(fileTextScanRangeParams); + return fileAttributes; } @Override - public void analyzeFileFormatProperties(Map formatProperties) + public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException { } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java index 208e842e0a55fa..0d51a9f121663f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java @@ -39,6 +39,6 @@ public void setUp() { public void testAnalyzeFileFormatProperties() { Map properties = new HashMap<>(); // Add properties if needed - checker.analyzeFileFormatProperties(properties); + checker.analyzeFileFormatProperties(properties, true); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java index c4c2b876887fe7..83f224d13f986f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java @@ -44,7 +44,7 @@ public void testAnalyzeFileFormatPropertiesValid() throws AnalysisException { properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, "\n"); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "1"); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(",", csvConfigurator.getColumnSeparator()); Assert.assertEquals("\n", csvConfigurator.getLineDelimiter()); @@ -57,7 +57,7 @@ public void testAnalyzeFileFormatPropertiesInvalidSeparator() { properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ""); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); }); } @@ -67,7 +67,7 @@ public void testAnalyzeFileFormatPropertiesInvalidLineDelimiter() { properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, ""); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); }); } @@ -77,7 +77,7 @@ public void testAnalyzeFileFormatPropertiesInvalidEnclose() { properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, "invalid"); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); }); } @@ -86,7 +86,7 @@ public void testAnalyzeFileFormatPropertiesValidEnclose() throws AnalysisExcepti Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, "\""); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals((byte) '"', csvConfigurator.getEnclose()); } @@ -96,7 +96,7 @@ public void testAnalyzeFileFormatPropertiesSkipLinesNegative() { properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "-1"); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); }); } @@ -105,7 +105,7 @@ public void testAnalyzeFileFormatPropertiesSkipLinesLargeValue() throws Analysis Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "1000"); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(1000, csvConfigurator.getSkipLines()); } @@ -114,7 +114,7 @@ public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesTrue() throws Analysi Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, csvConfigurator.isTrimDoubleQuotes()); } @@ -123,7 +123,7 @@ public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesFalse() throws Analys Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "false"); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, csvConfigurator.isTrimDoubleQuotes()); } @@ -133,16 +133,16 @@ public void testAnalyzeFileFormatPropertiesInvalidCompressType() { properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COMPRESS_TYPE, "invalid"); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); }); } @Test public void testAnalyzeFileFormatPropertiesValidCompressType() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COMPRESS_TYPE, "gzip"); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COMPRESS_TYPE, "gz"); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(TFileCompressType.GZ, csvConfigurator.getCompressionType()); } @@ -150,19 +150,7 @@ public void testAnalyzeFileFormatPropertiesValidCompressType() throws AnalysisEx public void testAnalyzeFileFormatPropertiesEmptyCsvSchema() { Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, ""); - - Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); - }); - } - - @Test - public void testAnalyzeFileFormatPropertiesValidCsvSchema() throws AnalysisException { - Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, "column1,column2,column3"); - - csvConfigurator.analyzeFileFormatProperties(properties); - Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); + csvConfigurator.analyzeFileFormatProperties(properties, true); } @Test @@ -171,7 +159,7 @@ public void testAnalyzeFileFormatPropertiesValidEncloseMultipleCharacters() { properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, "\"\""); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); }); } @@ -180,9 +168,8 @@ public void testAnalyzeFileFormatPropertiesValidEncloseEmpty() { Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, ""); - Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); - }); + csvConfigurator.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(0, csvConfigurator.getEnclose()); } @Test @@ -190,8 +177,8 @@ public void testAnalyzeFileFormatPropertiesSkipLinesAsString() { Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "abc"); - Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); + Assert.assertThrows(NumberFormatException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties, true); }); } @@ -200,7 +187,7 @@ public void testAnalyzeFileFormatPropertiesValidColumnSeparator() throws Analysi Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ";"); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(";", csvConfigurator.getColumnSeparator()); } @@ -208,10 +195,7 @@ public void testAnalyzeFileFormatPropertiesValidColumnSeparator() throws Analysi public void testAnalyzeFileFormatPropertiesLineDelimiterAsString() { Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, "abc"); - - Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); - }); + csvConfigurator.analyzeFileFormatProperties(properties, true); } @Test @@ -219,7 +203,7 @@ public void testAnalyzeFileFormatPropertiesValidLineDelimiter() throws AnalysisE Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, "\r\n"); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("\r\n", csvConfigurator.getLineDelimiter()); } @@ -228,7 +212,7 @@ public void testAnalyzeFileFormatPropertiesValidTrimDoubleQuotes() throws Analys Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, csvConfigurator.isTrimDoubleQuotes()); } @@ -237,17 +221,17 @@ public void testAnalyzeFileFormatPropertiesInvalidTrimDoubleQuotes() { Map properties = new HashMap<>(); properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "invalid"); - Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties); - }); + csvConfigurator.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(false, csvConfigurator.isTrimDoubleQuotes()); } @Test public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpaces() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, " column1 , column2 , column3 "); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, + " column1:int ; column2:string ; column3:double "); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); Assert.assertEquals("column1", csvConfigurator.getCsvSchema().get(0).getName().trim()); Assert.assertEquals("column2", csvConfigurator.getCsvSchema().get(1).getName().trim()); @@ -257,12 +241,43 @@ public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpaces() throws Ana @Test public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpecialCharacters() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, "col1@#$,col2&*(),col3"); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, + "col1@#$:int;col2&*:string;col3:double"); - csvConfigurator.analyzeFileFormatProperties(properties); + csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); Assert.assertEquals("col1@#$", csvConfigurator.getCsvSchema().get(0).getName()); - Assert.assertEquals("col2&*()", csvConfigurator.getCsvSchema().get(1).getName()); + Assert.assertEquals("col2&*", csvConfigurator.getCsvSchema().get(1).getName()); Assert.assertEquals("col3", csvConfigurator.getCsvSchema().get(2).getName()); } + + @Test + public void testAnalyzeFileFormatPropertiesInvalidCsvSchema() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, "column1,column2,column3"); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties, true); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesInvalidCsvSchemaWithSpecialCharacters() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, + "col1@#$:int;col2&*():string;col3:double"); + + Assert.assertThrows(AnalysisException.class, () -> { + csvConfigurator.analyzeFileFormatProperties(properties, true); + }); + } + + @Test + public void testAnalyzeFileFormatPropertiesValidCsvSchema() throws AnalysisException { + Map properties = new HashMap<>(); + properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, + "column1:int;column2:string;column3:double"); + csvConfigurator.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java index 40cb9f0f58dfff..52da892b2587c0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java @@ -39,7 +39,7 @@ public void setUp() { @Test public void testAnalyzeFileFormatPropertiesEmpty() throws AnalysisException { Map properties = new HashMap<>(); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("", configurator.getJsonRoot()); Assert.assertEquals("", configurator.getJsonPaths()); @@ -54,7 +54,7 @@ public void testAnalyzeFileFormatPropertiesValidJsonRoot() throws AnalysisExcept Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_ROOT, "data.items"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("data.items", configurator.getJsonRoot()); } @@ -64,7 +64,7 @@ public void testAnalyzeFileFormatPropertiesValidJsonPaths() throws AnalysisExcep properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, "[\"$.name\", \"$.age\", \"$.city\"]"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("[\"$.name\", \"$.age\", \"$.city\"]", configurator.getJsonPaths()); } @@ -73,7 +73,7 @@ public void testAnalyzeFileFormatPropertiesStripOuterArrayTrue() throws Analysis Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "true"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, configurator.isStripOuterArray()); } @@ -82,7 +82,7 @@ public void testAnalyzeFileFormatPropertiesStripOuterArrayFalse() throws Analysi Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "false"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, configurator.isStripOuterArray()); } @@ -91,7 +91,7 @@ public void testAnalyzeFileFormatPropertiesReadJsonByLineTrue() throws AnalysisE Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "true"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, configurator.isReadJsonByLine()); } @@ -100,7 +100,7 @@ public void testAnalyzeFileFormatPropertiesReadJsonByLineFalse() throws Analysis Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "false"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, configurator.isReadJsonByLine()); } @@ -109,7 +109,7 @@ public void testAnalyzeFileFormatPropertiesNumAsStringTrue() throws AnalysisExce Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_NUM_AS_STRING, "true"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, configurator.isNumAsString()); } @@ -118,7 +118,7 @@ public void testAnalyzeFileFormatPropertiesNumAsStringFalse() throws AnalysisExc Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_NUM_AS_STRING, "false"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, configurator.isNumAsString()); } @@ -127,7 +127,7 @@ public void testAnalyzeFileFormatPropertiesFuzzyParseTrue() throws AnalysisExcep Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "true"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, configurator.isFuzzyParse()); } @@ -136,7 +136,7 @@ public void testAnalyzeFileFormatPropertiesFuzzyParseFalse() throws AnalysisExce Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, configurator.isFuzzyParse()); } @@ -145,7 +145,7 @@ public void testAnalyzeFileFormatPropertiesInvalidBooleanValue() throws Analysis Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "invalid"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, configurator.isFuzzyParse()); } @@ -159,7 +159,7 @@ public void testAnalyzeFileFormatPropertiesAllProperties() throws AnalysisExcept properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_NUM_AS_STRING, "true"); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "true"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("data.records", configurator.getJsonRoot()); Assert.assertEquals("[\"$.id\", \"$.name\"]", configurator.getJsonPaths()); @@ -174,7 +174,7 @@ public void testAnalyzeFileFormatPropertiesSpecialCharactersInJsonRoot() throws Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_ROOT, "data.special@#$%^&*()"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("data.special@#$%^&*()", configurator.getJsonRoot()); } @@ -184,7 +184,7 @@ public void testAnalyzeFileFormatPropertiesComplexJsonPaths() throws AnalysisExc properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, "[\"$.deeply.nested[0].array[*].field\", \"$.complex.path[?(@.type=='value')]\"]"); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("[\"$.deeply.nested[0].array[*].field\", \"$.complex.path[?(@.type=='value')]\"]", configurator.getJsonPaths()); } @@ -194,7 +194,7 @@ public void testAnalyzeFileFormatPropertiesEmptyJsonPaths() throws AnalysisExcep Map properties = new HashMap<>(); properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, ""); - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("", configurator.getJsonPaths()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java index 1283864e6e4141..77b3fced7c7551 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java @@ -40,7 +40,7 @@ public void setUp() { public void testAnalyzeFileFormatProperties() { Map properties = new HashMap<>(); // Add properties if needed - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(TFileCompressType.ZLIB, configurator.getOrcCompressionType()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java index 3b217ba61bb1f3..c7689881f21879 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java @@ -40,7 +40,7 @@ public void setUp() { public void testAnalyzeFileFormatProperties() { Map properties = new HashMap<>(); // Add properties if needed - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(TParquetCompressionType.SNAPPY, configurator.getParquetCompressionType()); Assert.assertEquals(false, configurator.isParquetDisableDictionary()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java index 25451122da0685..6c34851b16000f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java @@ -38,6 +38,6 @@ public void setUp() { public void testAnalyzeFileFormatProperties() { Map properties = new HashMap<>(); // Add properties if needed - configurator.analyzeFileFormatProperties(properties); + configurator.analyzeFileFormatProperties(properties, true); } } From 00c292e4c88bc8c59ca818194a31f5148ef8107a Mon Sep 17 00:00:00 2001 From: BePPPower Date: Wed, 23 Apr 2025 15:06:01 +0800 Subject: [PATCH 3/7] fix 3 --- .../property/constants/AvroProperties.java | 21 ++++++ .../property/constants/CsvProperties.java | 34 +++++++++ .../constants/FileFormatBaseProperties.java | 32 +++++++++ .../property/constants/JsonProperties.java | 27 ++++++++ .../property/constants/OrcProperties.java | 22 ++++++ .../property/constants/ParquetProperties.java | 26 +++++++ .../property/constants/WalProperties.java | 21 ++++++ ...tor.java => AvroFileFormatProperties.java} | 14 ++-- ...ator.java => CsvFileFormatProperties.java} | 50 +++++--------- ...gurator.java => FileFormatProperties.java} | 69 ++++++++----------- ...tor.java => JsonFileFormatProperties.java} | 14 +--- ...ator.java => OrcFileFormatProperties.java} | 7 +- ....java => ParquetFileFormatProperties.java} | 11 +-- ...ator.java => WalFileFormatProperties.java} | 14 ++-- ...java => AvroFileFormatPropertiesTest.java} | 6 +- ....java => CsvFileFormatPropertiesTest.java} | 61 ++++++++-------- ...est.java => FileFormatPropertiesTest.java} | 4 +- ...java => JsonFileFormatPropertiesTest.java} | 47 ++++++------- ....java => OrcFileFormatPropertiesTest.java} | 6 +- ...a => ParquetFileFormatPropertiesTest.java} | 6 +- ....java => WalFileFormatPropertiesTest.java} | 6 +- 21 files changed, 318 insertions(+), 180 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AvroProperties.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CsvProperties.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/FileFormatBaseProperties.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/JsonProperties.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OrcProperties.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ParquetProperties.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/WalProperties.java rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/{WalFileFormatConfigurator.java => AvroFileFormatProperties.java} (93%) rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/{CsvFileFormatConfigurator.java => CsvFileFormatProperties.java} (75%) rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/{FileFormatConfigurator.java => FileFormatProperties.java} (53%) rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/{JsonFileFormatConfigurator.java => JsonFileFormatProperties.java} (87%) rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/{OrcFileFormatConfigurator.java => OrcFileFormatProperties.java} (91%) rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/{ParquetFileFormatConfigurator.java => ParquetFileFormatProperties.java} (82%) rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/{AvroFileFormatConfigurator.java => WalFileFormatProperties.java} (93%) rename fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/{AvroFileFormatConfiguratorTest.java => AvroFileFormatPropertiesTest.java} (88%) rename fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/{CsvFileFormatConfiguratorTest.java => CsvFileFormatPropertiesTest.java} (76%) rename fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/{FileFormatConfiguratorTest.java => FileFormatPropertiesTest.java} (90%) rename fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/{JsonFileFormatConfiguratorTest.java => JsonFileFormatPropertiesTest.java} (75%) rename fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/{OrcFileFormatConfiguratorTest.java => OrcFileFormatPropertiesTest.java} (89%) rename fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/{ParquetFileFormatConfiguratorTest.java => ParquetFileFormatPropertiesTest.java} (88%) rename fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/{WalFileFormatConfiguratorTest.java => WalFileFormatPropertiesTest.java} (87%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AvroProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AvroProperties.java new file mode 100644 index 00000000000000..80846307ffe95d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AvroProperties.java @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.constants; + +public class AvroProperties { +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CsvProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CsvProperties.java new file mode 100644 index 00000000000000..df54022402a417 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CsvProperties.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.constants; + +public class CsvProperties { + public static final String DEFAULT_COLUMN_SEPARATOR = "\t"; + public static final String DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR = "\001"; + public static final String DEFAULT_LINE_DELIMITER = "\n"; + + public static final String PROP_COLUMN_SEPARATOR = "column_separator"; + public static final String PROP_LINE_DELIMITER = "line_delimiter"; + + public static final String PROP_SKIP_LINES = "skip_lines"; + public static final String PROP_CSV_SCHEMA = "csv_schema"; + public static final String PROP_COMPRESS_TYPE = "compress_type"; + public static final String PROP_TRIM_DOUBLE_QUOTES = "trim_double_quotes"; + + public static final String PROP_ENCLOSE = "enclose"; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/FileFormatBaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/FileFormatBaseProperties.java new file mode 100644 index 00000000000000..19a29432732181 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/FileFormatBaseProperties.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.constants; + +public class FileFormatBaseProperties { + public static final String PROP_FORMAT = "format"; + public static final String FORMAT_PARQUET = "parquet"; + public static final String FORMAT_CSV = "csv"; + public static final String FORMAT_CSV_WITH_NAMES = "csv_with_names"; + public static final String FORMAT_CSV_WITH_NAMES_AND_TYPES = "csv_with_names_and_types"; + public static final String FORMAT_HIVE_TEXT = "hive_text"; + public static final String FORMAT_ORC = "orc"; + public static final String FORMAT_JSON = "json"; + public static final String FORMAT_AVRO = "avro"; + public static final String FORMAT_WAL = "wal"; + public static final String FORMAT_ARROW = "arrow"; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/JsonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/JsonProperties.java new file mode 100644 index 00000000000000..7b48a29a14a355 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/JsonProperties.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.constants; + +public class JsonProperties { + public static final String PROP_JSON_ROOT = "json_root"; + public static final String PROP_JSON_PATHS = "jsonpaths"; + public static final String PROP_STRIP_OUTER_ARRAY = "strip_outer_array"; + public static final String PROP_READ_JSON_BY_LINE = "read_json_by_line"; + public static final String PROP_NUM_AS_STRING = "num_as_string"; + public static final String PROP_FUZZY_PARSE = "fuzzy_parse"; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OrcProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OrcProperties.java new file mode 100644 index 00000000000000..797c71e76f0d69 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OrcProperties.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.constants; + +public class OrcProperties { + public static final String COMPRESS_TYPE = "compress_type"; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ParquetProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ParquetProperties.java new file mode 100644 index 00000000000000..6685566ac25cf8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ParquetProperties.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.constants; + +import org.apache.doris.thrift.TParquetVersion; + +public class ParquetProperties { + public static final String PARQUET_DISABLE_DICTIONARY = "disable_dictionary"; + public static final TParquetVersion parquetVersion = TParquetVersion.PARQUET_1_0; + public static final String PARQUET_VERSION = "version"; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/WalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/WalProperties.java new file mode 100644 index 00000000000000..9b535132fa624b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/WalProperties.java @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.constants; + +public class WalProperties { +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java similarity index 93% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java index 7b64a2e9eb2c4e..19c77b33646e57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java @@ -26,11 +26,16 @@ import java.util.Map; -public class WalFileFormatConfigurator extends FileFormatConfigurator { - public WalFileFormatConfigurator(TFileFormatType fileFormatType) { +public class AvroFileFormatProperties extends FileFormatProperties { + public AvroFileFormatProperties(TFileFormatType fileFormatType) { super(fileFormatType); } + @Override + public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) + throws AnalysisException { + } + @Override public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { return null; @@ -48,9 +53,4 @@ public TFileAttributes toTFileAttributes() { fileAttributes.setTextParams(fileTextScanRangeParams); return fileAttributes; } - - @Override - public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) - throws AnalysisException { - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java similarity index 75% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java index f863bb813e5ce8..cd4479f57abe90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.common.util.FileFormatUtils; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.property.constants.CsvProperties; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.qe.ConnectContext; @@ -38,13 +39,14 @@ import java.util.List; import java.util.Map; -public class CsvFileFormatConfigurator extends FileFormatConfigurator { - public static final Logger LOG = LogManager.getLogger(CsvFileFormatConfigurator.class); +public class CsvFileFormatProperties extends FileFormatProperties { + public static final Logger LOG = LogManager.getLogger( + org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties.class); private String headerType = ""; private TTextSerdeType textSerdeType = TTextSerdeType.JSON_TEXT_SERDE; - private String columnSeparator = CsvFileFormatProperties.DEFAULT_COLUMN_SEPARATOR; - private String lineDelimiter = CsvFileFormatProperties.DEFAULT_LINE_DELIMITER; + private String columnSeparator = CsvProperties.DEFAULT_COLUMN_SEPARATOR; + private String lineDelimiter = CsvProperties.DEFAULT_LINE_DELIMITER; private boolean trimDoubleQuotes; private int skipLines; private byte enclose; @@ -53,20 +55,20 @@ public class CsvFileFormatConfigurator extends FileFormatConfigurator { // User specified csv columns, it will override columns got from file private final List csvSchema = Lists.newArrayList(); - String defaultColumnSeparator = CsvFileFormatProperties.DEFAULT_COLUMN_SEPARATOR; + String defaultColumnSeparator = CsvProperties.DEFAULT_COLUMN_SEPARATOR; - public CsvFileFormatConfigurator(TFileFormatType fileFormatType) { + public CsvFileFormatProperties(TFileFormatType fileFormatType) { super(fileFormatType); } - public CsvFileFormatConfigurator(TFileFormatType fileFormatType, String defaultColumnSeparator, + public CsvFileFormatProperties(TFileFormatType fileFormatType, String defaultColumnSeparator, TTextSerdeType textSerdeType) { super(fileFormatType); this.defaultColumnSeparator = defaultColumnSeparator; this.textSerdeType = textSerdeType; } - public CsvFileFormatConfigurator(TFileFormatType fileFormatType, String headerType) { + public CsvFileFormatProperties(TFileFormatType fileFormatType, String headerType) { super(fileFormatType); this.headerType = headerType; } @@ -77,21 +79,21 @@ public void analyzeFileFormatProperties(Map formatProperties, bo throws AnalysisException { try { // check properties specified by user -- formatProperties - columnSeparator = getOrDefaultAndRemove(formatProperties, CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, + columnSeparator = getOrDefaultAndRemove(formatProperties, CsvProperties.PROP_COLUMN_SEPARATOR, defaultColumnSeparator, isRemoveOriginProperty); if (Strings.isNullOrEmpty(columnSeparator)) { throw new AnalysisException("column_separator can not be empty."); } columnSeparator = Separator.convertSeparator(columnSeparator); - lineDelimiter = getOrDefaultAndRemove(formatProperties, CsvFileFormatProperties.PROP_LINE_DELIMITER, - CsvFileFormatProperties.DEFAULT_LINE_DELIMITER, isRemoveOriginProperty); + lineDelimiter = getOrDefaultAndRemove(formatProperties, CsvProperties.PROP_LINE_DELIMITER, + CsvProperties.DEFAULT_LINE_DELIMITER, isRemoveOriginProperty); if (Strings.isNullOrEmpty(lineDelimiter)) { throw new AnalysisException("line_delimiter can not be empty."); } lineDelimiter = Separator.convertSeparator(lineDelimiter); - String enclosedString = getOrDefaultAndRemove(formatProperties, CsvFileFormatProperties.PROP_ENCLOSE, + String enclosedString = getOrDefaultAndRemove(formatProperties, CsvProperties.PROP_ENCLOSE, "", isRemoveOriginProperty); if (!Strings.isNullOrEmpty(enclosedString)) { if (enclosedString.length() > 1) { @@ -104,16 +106,16 @@ public void analyzeFileFormatProperties(Map formatProperties, bo } trimDoubleQuotes = Boolean.valueOf(getOrDefaultAndRemove(formatProperties, - CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "", isRemoveOriginProperty)) + CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "", isRemoveOriginProperty)) .booleanValue(); skipLines = Integer.valueOf(getOrDefaultAndRemove(formatProperties, - CsvFileFormatProperties.PROP_SKIP_LINES, "0", isRemoveOriginProperty)).intValue(); + CsvProperties.PROP_SKIP_LINES, "0", isRemoveOriginProperty)).intValue(); if (skipLines < 0) { throw new AnalysisException("skipLines should not be less than 0."); } String compressTypeStr = getOrDefaultAndRemove(formatProperties, - CsvFileFormatProperties.PROP_COMPRESS_TYPE, "UNKNOWN", isRemoveOriginProperty); + CsvProperties.PROP_COMPRESS_TYPE, "UNKNOWN", isRemoveOriginProperty); try { compressionType = Util.getFileCompressType(compressTypeStr); } catch (IllegalArgumentException e) { @@ -121,7 +123,7 @@ public void analyzeFileFormatProperties(Map formatProperties, bo } FileFormatUtils.parseCsvSchema(csvSchema, getOrDefaultAndRemove(formatProperties, - CsvFileFormatProperties.PROP_CSV_SCHEMA, "", isRemoveOriginProperty)); + CsvProperties.PROP_CSV_SCHEMA, "", isRemoveOriginProperty)); if (LOG.isDebugEnabled()) { LOG.debug("get csv schema: {}", csvSchema); } @@ -190,20 +192,4 @@ public byte getEnclose() { public List getCsvSchema() { return csvSchema; } - - public static class CsvFileFormatProperties { - public static final String DEFAULT_COLUMN_SEPARATOR = "\t"; - public static final String DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR = "\001"; - public static final String DEFAULT_LINE_DELIMITER = "\n"; - - public static final String PROP_COLUMN_SEPARATOR = "column_separator"; - public static final String PROP_LINE_DELIMITER = "line_delimiter"; - - public static final String PROP_SKIP_LINES = "skip_lines"; - public static final String PROP_CSV_SCHEMA = "csv_schema"; - public static final String PROP_COMPRESS_TYPE = "compress_type"; - public static final String PROP_TRIM_DOUBLE_QUOTES = "trim_double_quotes"; - - public static final String PROP_ENCLOSE = "enclose"; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java similarity index 53% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java index cd4aa7f238da94..e8f835781a64db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java @@ -17,7 +17,8 @@ package org.apache.doris.datasource.property.fileformat; -import org.apache.doris.datasource.property.fileformat.CsvFileFormatConfigurator.CsvFileFormatProperties; +import org.apache.doris.datasource.property.constants.CsvProperties; +import org.apache.doris.datasource.property.constants.FileFormatBaseProperties; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; @@ -28,26 +29,12 @@ import java.util.Map; -public abstract class FileFormatConfigurator { - public static class FileFormatProperties { - public static final String PROP_FORMAT = "format"; - public static final String FORMAT_PARQUET = "parquet"; - public static final String FORMAT_CSV = "csv"; - public static final String FORMAT_CSV_WITH_NAMES = "csv_with_names"; - public static final String FORMAT_CSV_WITH_NAMES_AND_TYPES = "csv_with_names_and_types"; - public static final String FORMAT_HIVE_TEXT = "hive_text"; - public static final String FORMAT_ORC = "orc"; - public static final String FORMAT_JSON = "json"; - public static final String FORMAT_AVRO = "avro"; - public static final String FORMAT_WAL = "wal"; - public static final String FORMAT_ARROW = "arrow"; - } - +public abstract class FileFormatProperties { protected TFileFormatType fileFormatType; protected TFileCompressType compressionType; - public FileFormatConfigurator(TFileFormatType fileFormatType) { + public FileFormatProperties(TFileFormatType fileFormatType) { this.fileFormatType = fileFormatType; } @@ -74,38 +61,38 @@ public abstract void analyzeFileFormatProperties( public abstract TFileAttributes toTFileAttributes(); - public static FileFormatConfigurator createFileFormatChecker(String formatString) { + public static FileFormatProperties createFileFormatChecker(String formatString) { switch (formatString) { - case FileFormatProperties.FORMAT_CSV: - return new CsvFileFormatConfigurator(TFileFormatType.FORMAT_CSV_PLAIN); - case FileFormatProperties.FORMAT_HIVE_TEXT: - return new CsvFileFormatConfigurator(TFileFormatType.FORMAT_CSV_PLAIN, - CsvFileFormatProperties.DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR, + case FileFormatBaseProperties.FORMAT_CSV: + return new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN); + case FileFormatBaseProperties.FORMAT_HIVE_TEXT: + return new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN, + CsvProperties.DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR, TTextSerdeType.HIVE_TEXT_SERDE); - case FileFormatProperties.FORMAT_CSV_WITH_NAMES: - return new CsvFileFormatConfigurator(TFileFormatType.FORMAT_CSV_PLAIN, - FileFormatProperties.FORMAT_CSV_WITH_NAMES); - case FileFormatProperties.FORMAT_CSV_WITH_NAMES_AND_TYPES: - return new CsvFileFormatConfigurator(TFileFormatType.FORMAT_CSV_PLAIN, - FileFormatProperties.FORMAT_CSV_WITH_NAMES_AND_TYPES); - case FileFormatProperties.FORMAT_PARQUET: - return new ParquetFileFormatConfigurator(TFileFormatType.FORMAT_PARQUET); - case FileFormatProperties.FORMAT_ORC: - return new OrcFileFormatConfigurator(TFileFormatType.FORMAT_ORC); - case FileFormatProperties.FORMAT_JSON: - return new JsonFileFormatConfigurator(TFileFormatType.FORMAT_JSON); - case FileFormatProperties.FORMAT_AVRO: - return new AvroFileFormatConfigurator(TFileFormatType.FORMAT_AVRO); - case FileFormatProperties.FORMAT_WAL: - return new WalFileFormatConfigurator(TFileFormatType.FORMAT_WAL); + case FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES: + return new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN, + FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES); + case FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES_AND_TYPES: + return new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN, + FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES_AND_TYPES); + case FileFormatBaseProperties.FORMAT_PARQUET: + return new ParquetFileFormatProperties(TFileFormatType.FORMAT_PARQUET); + case FileFormatBaseProperties.FORMAT_ORC: + return new OrcFileFormatProperties(TFileFormatType.FORMAT_ORC); + case FileFormatBaseProperties.FORMAT_JSON: + return new JsonFileFormatProperties(TFileFormatType.FORMAT_JSON); + case FileFormatBaseProperties.FORMAT_AVRO: + return new AvroFileFormatProperties(TFileFormatType.FORMAT_AVRO); + case FileFormatBaseProperties.FORMAT_WAL: + return new WalFileFormatProperties(TFileFormatType.FORMAT_WAL); default: throw new AnalysisException("format:" + formatString + " is not supported."); } } - public static FileFormatConfigurator createFileFormatChecker(Map formatProperties) + public static FileFormatProperties createFileFormatChecker(Map formatProperties) throws AnalysisException { - String formatString = formatProperties.getOrDefault(FileFormatProperties.PROP_FORMAT, "") + String formatString = formatProperties.getOrDefault(FileFormatBaseProperties.PROP_FORMAT, "") .toLowerCase(); return createFileFormatChecker(formatString); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java similarity index 87% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java index c05bf24ec3ba9e..65cacf0c41f1c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java @@ -28,7 +28,7 @@ import java.util.Map; -public class JsonFileFormatConfigurator extends FileFormatConfigurator { +public class JsonFileFormatProperties extends FileFormatProperties { // from ExternalFileTableValuedFunction: private String jsonRoot = ""; private String jsonPaths = ""; @@ -38,7 +38,7 @@ public class JsonFileFormatConfigurator extends FileFormatConfigurator { private boolean fuzzyParse; - public JsonFileFormatConfigurator(TFileFormatType fileFormatType) { + public JsonFileFormatProperties(TFileFormatType fileFormatType) { super(fileFormatType); } @@ -119,14 +119,4 @@ public boolean isNumAsString() { public boolean isFuzzyParse() { return fuzzyParse; } - - - public static class JsonFileFormatProperties { - public static final String PROP_JSON_ROOT = "json_root"; - public static final String PROP_JSON_PATHS = "jsonpaths"; - public static final String PROP_STRIP_OUTER_ARRAY = "strip_outer_array"; - public static final String PROP_READ_JSON_BY_LINE = "read_json_by_line"; - public static final String PROP_NUM_AS_STRING = "num_as_string"; - public static final String PROP_FUZZY_PARSE = "fuzzy_parse"; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatProperties.java similarity index 91% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatProperties.java index 084c5a81d073fc..6deb5b22bf78d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatProperties.java @@ -27,10 +27,10 @@ import java.util.Map; -public class OrcFileFormatConfigurator extends FileFormatConfigurator { +public class OrcFileFormatProperties extends FileFormatProperties { private TFileCompressType orcCompressionType = TFileCompressType.ZLIB; - public OrcFileFormatConfigurator(TFileFormatType fileFormatType) { + public OrcFileFormatProperties(TFileFormatType fileFormatType) { super(fileFormatType); } @@ -60,7 +60,4 @@ public TFileAttributes toTFileAttributes() { public TFileCompressType getOrcCompressionType() { return orcCompressionType; } - - public static class OrcFileFormatProperties { - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java similarity index 82% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java index 6e22c789501dc6..e4b9dfe78f32ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java @@ -23,16 +23,15 @@ import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileTextScanRangeParams; import org.apache.doris.thrift.TParquetCompressionType; -import org.apache.doris.thrift.TParquetVersion; import org.apache.doris.thrift.TResultFileSinkOptions; import java.util.Map; -public class ParquetFileFormatConfigurator extends FileFormatConfigurator { +public class ParquetFileFormatProperties extends FileFormatProperties { private TParquetCompressionType parquetCompressionType = TParquetCompressionType.SNAPPY; private boolean parquetDisableDictionary = false; - public ParquetFileFormatConfigurator(TFileFormatType fileFormatType) { + public ParquetFileFormatProperties(TFileFormatType fileFormatType) { super(fileFormatType); } @@ -66,10 +65,4 @@ public TParquetCompressionType getParquetCompressionType() { public boolean isParquetDisableDictionary() { return parquetDisableDictionary; } - - public static class ParquetFileFormatProperties { - public static final String PARQUET_DISABLE_DICTIONARY = "disable_dictionary"; - public static final TParquetVersion parquetVersion = TParquetVersion.PARQUET_1_0; - public static final String PARQUET_VERSION = "version"; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatProperties.java similarity index 93% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatProperties.java index e5ad5da0f48059..f935fa38b34a40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfigurator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatProperties.java @@ -26,16 +26,11 @@ import java.util.Map; -public class AvroFileFormatConfigurator extends FileFormatConfigurator { - public AvroFileFormatConfigurator(TFileFormatType fileFormatType) { +public class WalFileFormatProperties extends FileFormatProperties { + public WalFileFormatProperties(TFileFormatType fileFormatType) { super(fileFormatType); } - @Override - public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) - throws AnalysisException { - } - @Override public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { return null; @@ -53,4 +48,9 @@ public TFileAttributes toTFileAttributes() { fileAttributes.setTextParams(fileTextScanRangeParams); return fileAttributes; } + + @Override + public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) + throws AnalysisException { + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java similarity index 88% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java index 0d51a9f121663f..c8f106d1a30b9c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java @@ -26,13 +26,13 @@ import java.util.Map; -public class AvroFileFormatConfiguratorTest { +public class AvroFileFormatPropertiesTest { - private AvroFileFormatConfigurator checker; + private AvroFileFormatProperties checker; @Before public void setUp() { - checker = new AvroFileFormatConfigurator(TFileFormatType.FORMAT_AVRO); + checker = new AvroFileFormatProperties(TFileFormatType.FORMAT_AVRO); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java similarity index 76% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java index 83f224d13f986f..12a419758817f9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.property.fileformat; +import org.apache.doris.datasource.property.constants.CsvProperties; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TFileFormatType; @@ -28,21 +29,21 @@ import java.util.HashMap; import java.util.Map; -public class CsvFileFormatConfiguratorTest { +public class CsvFileFormatPropertiesTest { - private CsvFileFormatConfigurator csvConfigurator; + private CsvFileFormatProperties csvConfigurator; @Before public void setUp() { - csvConfigurator = new CsvFileFormatConfigurator(TFileFormatType.FORMAT_CSV_PLAIN); + csvConfigurator = new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN); } @Test public void testAnalyzeFileFormatPropertiesValid() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ","); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, "\n"); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "1"); + properties.put(CsvProperties.PROP_COLUMN_SEPARATOR, ","); + properties.put(CsvProperties.PROP_LINE_DELIMITER, "\n"); + properties.put(CsvProperties.PROP_SKIP_LINES, "1"); csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -54,7 +55,7 @@ public void testAnalyzeFileFormatPropertiesValid() throws AnalysisException { @Test public void testAnalyzeFileFormatPropertiesInvalidSeparator() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ""); + properties.put(CsvProperties.PROP_COLUMN_SEPARATOR, ""); Assert.assertThrows(AnalysisException.class, () -> { csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -64,7 +65,7 @@ public void testAnalyzeFileFormatPropertiesInvalidSeparator() { @Test public void testAnalyzeFileFormatPropertiesInvalidLineDelimiter() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, ""); + properties.put(CsvProperties.PROP_LINE_DELIMITER, ""); Assert.assertThrows(AnalysisException.class, () -> { csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -74,7 +75,7 @@ public void testAnalyzeFileFormatPropertiesInvalidLineDelimiter() { @Test public void testAnalyzeFileFormatPropertiesInvalidEnclose() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, "invalid"); + properties.put(CsvProperties.PROP_ENCLOSE, "invalid"); Assert.assertThrows(AnalysisException.class, () -> { csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -84,7 +85,7 @@ public void testAnalyzeFileFormatPropertiesInvalidEnclose() { @Test public void testAnalyzeFileFormatPropertiesValidEnclose() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, "\""); + properties.put(CsvProperties.PROP_ENCLOSE, "\""); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals((byte) '"', csvConfigurator.getEnclose()); @@ -93,7 +94,7 @@ public void testAnalyzeFileFormatPropertiesValidEnclose() throws AnalysisExcepti @Test public void testAnalyzeFileFormatPropertiesSkipLinesNegative() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "-1"); + properties.put(CsvProperties.PROP_SKIP_LINES, "-1"); Assert.assertThrows(AnalysisException.class, () -> { csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -103,7 +104,7 @@ public void testAnalyzeFileFormatPropertiesSkipLinesNegative() { @Test public void testAnalyzeFileFormatPropertiesSkipLinesLargeValue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "1000"); + properties.put(CsvProperties.PROP_SKIP_LINES, "1000"); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(1000, csvConfigurator.getSkipLines()); @@ -112,7 +113,7 @@ public void testAnalyzeFileFormatPropertiesSkipLinesLargeValue() throws Analysis @Test public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesTrue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); + properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, csvConfigurator.isTrimDoubleQuotes()); @@ -121,7 +122,7 @@ public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesTrue() throws Analysi @Test public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesFalse() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "false"); + properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "false"); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, csvConfigurator.isTrimDoubleQuotes()); @@ -130,7 +131,7 @@ public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesFalse() throws Analys @Test public void testAnalyzeFileFormatPropertiesInvalidCompressType() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COMPRESS_TYPE, "invalid"); + properties.put(CsvProperties.PROP_COMPRESS_TYPE, "invalid"); Assert.assertThrows(AnalysisException.class, () -> { csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -140,7 +141,7 @@ public void testAnalyzeFileFormatPropertiesInvalidCompressType() { @Test public void testAnalyzeFileFormatPropertiesValidCompressType() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COMPRESS_TYPE, "gz"); + properties.put(CsvProperties.PROP_COMPRESS_TYPE, "gz"); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(TFileCompressType.GZ, csvConfigurator.getCompressionType()); @@ -149,14 +150,14 @@ public void testAnalyzeFileFormatPropertiesValidCompressType() throws AnalysisEx @Test public void testAnalyzeFileFormatPropertiesEmptyCsvSchema() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, ""); + properties.put(CsvProperties.PROP_CSV_SCHEMA, ""); csvConfigurator.analyzeFileFormatProperties(properties, true); } @Test public void testAnalyzeFileFormatPropertiesValidEncloseMultipleCharacters() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, "\"\""); + properties.put(CsvProperties.PROP_ENCLOSE, "\"\""); Assert.assertThrows(AnalysisException.class, () -> { csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -166,7 +167,7 @@ public void testAnalyzeFileFormatPropertiesValidEncloseMultipleCharacters() { @Test public void testAnalyzeFileFormatPropertiesValidEncloseEmpty() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_ENCLOSE, ""); + properties.put(CsvProperties.PROP_ENCLOSE, ""); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(0, csvConfigurator.getEnclose()); @@ -175,7 +176,7 @@ public void testAnalyzeFileFormatPropertiesValidEncloseEmpty() { @Test public void testAnalyzeFileFormatPropertiesSkipLinesAsString() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_SKIP_LINES, "abc"); + properties.put(CsvProperties.PROP_SKIP_LINES, "abc"); Assert.assertThrows(NumberFormatException.class, () -> { csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -185,7 +186,7 @@ public void testAnalyzeFileFormatPropertiesSkipLinesAsString() { @Test public void testAnalyzeFileFormatPropertiesValidColumnSeparator() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ";"); + properties.put(CsvProperties.PROP_COLUMN_SEPARATOR, ";"); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(";", csvConfigurator.getColumnSeparator()); @@ -194,14 +195,14 @@ public void testAnalyzeFileFormatPropertiesValidColumnSeparator() throws Analysi @Test public void testAnalyzeFileFormatPropertiesLineDelimiterAsString() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, "abc"); + properties.put(CsvProperties.PROP_LINE_DELIMITER, "abc"); csvConfigurator.analyzeFileFormatProperties(properties, true); } @Test public void testAnalyzeFileFormatPropertiesValidLineDelimiter() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_LINE_DELIMITER, "\r\n"); + properties.put(CsvProperties.PROP_LINE_DELIMITER, "\r\n"); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("\r\n", csvConfigurator.getLineDelimiter()); @@ -210,7 +211,7 @@ public void testAnalyzeFileFormatPropertiesValidLineDelimiter() throws AnalysisE @Test public void testAnalyzeFileFormatPropertiesValidTrimDoubleQuotes() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); + properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, csvConfigurator.isTrimDoubleQuotes()); @@ -219,7 +220,7 @@ public void testAnalyzeFileFormatPropertiesValidTrimDoubleQuotes() throws Analys @Test public void testAnalyzeFileFormatPropertiesInvalidTrimDoubleQuotes() { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "invalid"); + properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "invalid"); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, csvConfigurator.isTrimDoubleQuotes()); @@ -228,7 +229,7 @@ public void testAnalyzeFileFormatPropertiesInvalidTrimDoubleQuotes() { @Test public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpaces() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, + properties.put(CsvProperties.PROP_CSV_SCHEMA, " column1:int ; column2:string ; column3:double "); csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -241,7 +242,7 @@ public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpaces() throws Ana @Test public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpecialCharacters() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, + properties.put(CsvProperties.PROP_CSV_SCHEMA, "col1@#$:int;col2&*:string;col3:double"); csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -254,7 +255,7 @@ public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpecialCharacters() @Test public void testAnalyzeFileFormatPropertiesInvalidCsvSchema() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, "column1,column2,column3"); + properties.put(CsvProperties.PROP_CSV_SCHEMA, "column1,column2,column3"); Assert.assertThrows(AnalysisException.class, () -> { csvConfigurator.analyzeFileFormatProperties(properties, true); @@ -264,7 +265,7 @@ public void testAnalyzeFileFormatPropertiesInvalidCsvSchema() throws AnalysisExc @Test public void testAnalyzeFileFormatPropertiesInvalidCsvSchemaWithSpecialCharacters() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, + properties.put(CsvProperties.PROP_CSV_SCHEMA, "col1@#$:int;col2&*():string;col3:double"); Assert.assertThrows(AnalysisException.class, () -> { @@ -275,7 +276,7 @@ public void testAnalyzeFileFormatPropertiesInvalidCsvSchemaWithSpecialCharacters @Test public void testAnalyzeFileFormatPropertiesValidCsvSchema() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvFileFormatConfigurator.CsvFileFormatProperties.PROP_CSV_SCHEMA, + properties.put(CsvProperties.PROP_CSV_SCHEMA, "column1:int;column2:string;column3:double"); csvConfigurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatPropertiesTest.java similarity index 90% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatConfiguratorTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatPropertiesTest.java index d1f2f45092babb..ca9e9546fea7a9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatPropertiesTest.java @@ -23,12 +23,12 @@ import org.junit.Test; -public class FileFormatConfiguratorTest { +public class FileFormatPropertiesTest { @Test public void testCreateFileFormatCheckerInvalidFormat() { Assert.assertThrows(AnalysisException.class, () -> { - FileFormatConfigurator.createFileFormatChecker("invalid_format"); + FileFormatProperties.createFileFormatChecker("invalid_format"); }); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java similarity index 75% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java index 52da892b2587c0..d7e0be0ed57ba1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.property.fileformat; +import org.apache.doris.datasource.property.constants.JsonProperties; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.thrift.TFileFormatType; @@ -27,13 +28,13 @@ import java.util.HashMap; import java.util.Map; -public class JsonFileFormatConfiguratorTest { +public class JsonFileFormatPropertiesTest { - private JsonFileFormatConfigurator configurator; + private JsonFileFormatProperties configurator; @Before public void setUp() { - configurator = new JsonFileFormatConfigurator(TFileFormatType.FORMAT_JSON); + configurator = new JsonFileFormatProperties(TFileFormatType.FORMAT_JSON); } @Test @@ -52,7 +53,7 @@ public void testAnalyzeFileFormatPropertiesEmpty() throws AnalysisException { @Test public void testAnalyzeFileFormatPropertiesValidJsonRoot() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_ROOT, "data.items"); + properties.put(JsonProperties.PROP_JSON_ROOT, "data.items"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("data.items", configurator.getJsonRoot()); @@ -61,7 +62,7 @@ public void testAnalyzeFileFormatPropertiesValidJsonRoot() throws AnalysisExcept @Test public void testAnalyzeFileFormatPropertiesValidJsonPaths() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, + properties.put(JsonProperties.PROP_JSON_PATHS, "[\"$.name\", \"$.age\", \"$.city\"]"); configurator.analyzeFileFormatProperties(properties, true); @@ -71,7 +72,7 @@ public void testAnalyzeFileFormatPropertiesValidJsonPaths() throws AnalysisExcep @Test public void testAnalyzeFileFormatPropertiesStripOuterArrayTrue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "true"); + properties.put(JsonProperties.PROP_STRIP_OUTER_ARRAY, "true"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, configurator.isStripOuterArray()); @@ -80,7 +81,7 @@ public void testAnalyzeFileFormatPropertiesStripOuterArrayTrue() throws Analysis @Test public void testAnalyzeFileFormatPropertiesStripOuterArrayFalse() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "false"); + properties.put(JsonProperties.PROP_STRIP_OUTER_ARRAY, "false"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, configurator.isStripOuterArray()); @@ -89,7 +90,7 @@ public void testAnalyzeFileFormatPropertiesStripOuterArrayFalse() throws Analysi @Test public void testAnalyzeFileFormatPropertiesReadJsonByLineTrue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "true"); + properties.put(JsonProperties.PROP_READ_JSON_BY_LINE, "true"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, configurator.isReadJsonByLine()); @@ -98,7 +99,7 @@ public void testAnalyzeFileFormatPropertiesReadJsonByLineTrue() throws AnalysisE @Test public void testAnalyzeFileFormatPropertiesReadJsonByLineFalse() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "false"); + properties.put(JsonProperties.PROP_READ_JSON_BY_LINE, "false"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, configurator.isReadJsonByLine()); @@ -107,7 +108,7 @@ public void testAnalyzeFileFormatPropertiesReadJsonByLineFalse() throws Analysis @Test public void testAnalyzeFileFormatPropertiesNumAsStringTrue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_NUM_AS_STRING, "true"); + properties.put(JsonProperties.PROP_NUM_AS_STRING, "true"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, configurator.isNumAsString()); @@ -116,7 +117,7 @@ public void testAnalyzeFileFormatPropertiesNumAsStringTrue() throws AnalysisExce @Test public void testAnalyzeFileFormatPropertiesNumAsStringFalse() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_NUM_AS_STRING, "false"); + properties.put(JsonProperties.PROP_NUM_AS_STRING, "false"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, configurator.isNumAsString()); @@ -125,7 +126,7 @@ public void testAnalyzeFileFormatPropertiesNumAsStringFalse() throws AnalysisExc @Test public void testAnalyzeFileFormatPropertiesFuzzyParseTrue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "true"); + properties.put(JsonProperties.PROP_FUZZY_PARSE, "true"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, configurator.isFuzzyParse()); @@ -134,7 +135,7 @@ public void testAnalyzeFileFormatPropertiesFuzzyParseTrue() throws AnalysisExcep @Test public void testAnalyzeFileFormatPropertiesFuzzyParseFalse() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"); + properties.put(JsonProperties.PROP_FUZZY_PARSE, "false"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, configurator.isFuzzyParse()); @@ -143,7 +144,7 @@ public void testAnalyzeFileFormatPropertiesFuzzyParseFalse() throws AnalysisExce @Test public void testAnalyzeFileFormatPropertiesInvalidBooleanValue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "invalid"); + properties.put(JsonProperties.PROP_FUZZY_PARSE, "invalid"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, configurator.isFuzzyParse()); @@ -152,12 +153,12 @@ public void testAnalyzeFileFormatPropertiesInvalidBooleanValue() throws Analysis @Test public void testAnalyzeFileFormatPropertiesAllProperties() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_ROOT, "data.records"); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, "[\"$.id\", \"$.name\"]"); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "true"); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "true"); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_NUM_AS_STRING, "true"); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_FUZZY_PARSE, "true"); + properties.put(JsonProperties.PROP_JSON_ROOT, "data.records"); + properties.put(JsonProperties.PROP_JSON_PATHS, "[\"$.id\", \"$.name\"]"); + properties.put(JsonProperties.PROP_STRIP_OUTER_ARRAY, "true"); + properties.put(JsonProperties.PROP_READ_JSON_BY_LINE, "true"); + properties.put(JsonProperties.PROP_NUM_AS_STRING, "true"); + properties.put(JsonProperties.PROP_FUZZY_PARSE, "true"); configurator.analyzeFileFormatProperties(properties, true); @@ -172,7 +173,7 @@ public void testAnalyzeFileFormatPropertiesAllProperties() throws AnalysisExcept @Test public void testAnalyzeFileFormatPropertiesSpecialCharactersInJsonRoot() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_ROOT, "data.special@#$%^&*()"); + properties.put(JsonProperties.PROP_JSON_ROOT, "data.special@#$%^&*()"); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("data.special@#$%^&*()", configurator.getJsonRoot()); @@ -181,7 +182,7 @@ public void testAnalyzeFileFormatPropertiesSpecialCharactersInJsonRoot() throws @Test public void testAnalyzeFileFormatPropertiesComplexJsonPaths() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, + properties.put(JsonProperties.PROP_JSON_PATHS, "[\"$.deeply.nested[0].array[*].field\", \"$.complex.path[?(@.type=='value')]\"]"); configurator.analyzeFileFormatProperties(properties, true); @@ -192,7 +193,7 @@ public void testAnalyzeFileFormatPropertiesComplexJsonPaths() throws AnalysisExc @Test public void testAnalyzeFileFormatPropertiesEmptyJsonPaths() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonFileFormatConfigurator.JsonFileFormatProperties.PROP_JSON_PATHS, ""); + properties.put(JsonProperties.PROP_JSON_PATHS, ""); configurator.analyzeFileFormatProperties(properties, true); Assert.assertEquals("", configurator.getJsonPaths()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java similarity index 89% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java index 77b3fced7c7551..8769c51217ef29 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java @@ -27,13 +27,13 @@ import java.util.HashMap; import java.util.Map; -public class OrcFileFormatConfiguratorTest { +public class OrcFileFormatPropertiesTest { - private OrcFileFormatConfigurator configurator; + private OrcFileFormatProperties configurator; @Before public void setUp() { - configurator = new OrcFileFormatConfigurator(TFileFormatType.FORMAT_ORC); + configurator = new OrcFileFormatProperties(TFileFormatType.FORMAT_ORC); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java similarity index 88% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java index c7689881f21879..b028b19e72dac1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java @@ -27,13 +27,13 @@ import java.util.HashMap; import java.util.Map; -public class ParquetFileFormatConfiguratorTest { +public class ParquetFileFormatPropertiesTest { - private ParquetFileFormatConfigurator configurator; + private ParquetFileFormatProperties configurator; @Before public void setUp() { - configurator = new ParquetFileFormatConfigurator(TFileFormatType.FORMAT_PARQUET); + configurator = new ParquetFileFormatProperties(TFileFormatType.FORMAT_PARQUET); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java similarity index 87% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java index 6c34851b16000f..c7f269911941d2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatConfiguratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java @@ -25,13 +25,13 @@ import java.util.HashMap; import java.util.Map; -public class WalFileFormatConfiguratorTest { +public class WalFileFormatPropertiesTest { - private WalFileFormatConfigurator configurator; + private WalFileFormatProperties configurator; @Before public void setUp() { - configurator = new WalFileFormatConfigurator(TFileFormatType.FORMAT_WAL); + configurator = new WalFileFormatProperties(TFileFormatType.FORMAT_WAL); } @Test From 040aa749c767a91a95bb2ef2a56851961e2646a8 Mon Sep 17 00:00:00 2001 From: BePPPower Date: Wed, 23 Apr 2025 17:30:15 +0800 Subject: [PATCH 4/7] fix 4 --- .../fileformat/FileFormatProperties.java | 6 +- .../AvroFileFormatPropertiesTest.java | 6 +- .../CsvFileFormatPropertiesTest.java | 98 +++++++++---------- .../fileformat/FileFormatPropertiesTest.java | 4 +- .../JsonFileFormatPropertiesTest.java | 88 ++++++++--------- .../OrcFileFormatPropertiesTest.java | 8 +- .../ParquetFileFormatPropertiesTest.java | 10 +- .../WalFileFormatPropertiesTest.java | 6 +- 8 files changed, 113 insertions(+), 113 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java index e8f835781a64db..e5cd97613e070a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java @@ -61,7 +61,7 @@ public abstract void analyzeFileFormatProperties( public abstract TFileAttributes toTFileAttributes(); - public static FileFormatProperties createFileFormatChecker(String formatString) { + public static FileFormatProperties createFileFormatProperties(String formatString) { switch (formatString) { case FileFormatBaseProperties.FORMAT_CSV: return new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN); @@ -90,11 +90,11 @@ public static FileFormatProperties createFileFormatChecker(String formatString) } } - public static FileFormatProperties createFileFormatChecker(Map formatProperties) + public static FileFormatProperties createFileFormatProperties(Map formatProperties) throws AnalysisException { String formatString = formatProperties.getOrDefault(FileFormatBaseProperties.PROP_FORMAT, "") .toLowerCase(); - return createFileFormatChecker(formatString); + return createFileFormatProperties(formatString); } protected String getOrDefaultAndRemove(Map props, String key, String defaultValue, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java index c8f106d1a30b9c..b1885a1e46ce8b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java @@ -28,17 +28,17 @@ public class AvroFileFormatPropertiesTest { - private AvroFileFormatProperties checker; + private AvroFileFormatProperties avroFileFormatProperties; @Before public void setUp() { - checker = new AvroFileFormatProperties(TFileFormatType.FORMAT_AVRO); + avroFileFormatProperties = new AvroFileFormatProperties(TFileFormatType.FORMAT_AVRO); } @Test public void testAnalyzeFileFormatProperties() { Map properties = new HashMap<>(); // Add properties if needed - checker.analyzeFileFormatProperties(properties, true); + avroFileFormatProperties.analyzeFileFormatProperties(properties, true); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java index 12a419758817f9..34654c18718c46 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java @@ -31,11 +31,11 @@ public class CsvFileFormatPropertiesTest { - private CsvFileFormatProperties csvConfigurator; + private CsvFileFormatProperties csvFileFormatProperties; @Before public void setUp() { - csvConfigurator = new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN); + csvFileFormatProperties = new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN); } @Test @@ -45,11 +45,11 @@ public void testAnalyzeFileFormatPropertiesValid() throws AnalysisException { properties.put(CsvProperties.PROP_LINE_DELIMITER, "\n"); properties.put(CsvProperties.PROP_SKIP_LINES, "1"); - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(",", csvConfigurator.getColumnSeparator()); - Assert.assertEquals("\n", csvConfigurator.getLineDelimiter()); - Assert.assertEquals(1, csvConfigurator.getSkipLines()); + Assert.assertEquals(",", csvFileFormatProperties.getColumnSeparator()); + Assert.assertEquals("\n", csvFileFormatProperties.getLineDelimiter()); + Assert.assertEquals(1, csvFileFormatProperties.getSkipLines()); } @Test @@ -58,7 +58,7 @@ public void testAnalyzeFileFormatPropertiesInvalidSeparator() { properties.put(CsvProperties.PROP_COLUMN_SEPARATOR, ""); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); }); } @@ -68,7 +68,7 @@ public void testAnalyzeFileFormatPropertiesInvalidLineDelimiter() { properties.put(CsvProperties.PROP_LINE_DELIMITER, ""); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); }); } @@ -78,7 +78,7 @@ public void testAnalyzeFileFormatPropertiesInvalidEnclose() { properties.put(CsvProperties.PROP_ENCLOSE, "invalid"); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); }); } @@ -87,8 +87,8 @@ public void testAnalyzeFileFormatPropertiesValidEnclose() throws AnalysisExcepti Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_ENCLOSE, "\""); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals((byte) '"', csvConfigurator.getEnclose()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals((byte) '"', csvFileFormatProperties.getEnclose()); } @Test @@ -97,7 +97,7 @@ public void testAnalyzeFileFormatPropertiesSkipLinesNegative() { properties.put(CsvProperties.PROP_SKIP_LINES, "-1"); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); }); } @@ -106,8 +106,8 @@ public void testAnalyzeFileFormatPropertiesSkipLinesLargeValue() throws Analysis Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_SKIP_LINES, "1000"); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(1000, csvConfigurator.getSkipLines()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(1000, csvFileFormatProperties.getSkipLines()); } @Test @@ -115,8 +115,8 @@ public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesTrue() throws Analysi Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(true, csvConfigurator.isTrimDoubleQuotes()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(true, csvFileFormatProperties.isTrimDoubleQuotes()); } @Test @@ -124,8 +124,8 @@ public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesFalse() throws Analys Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "false"); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(false, csvConfigurator.isTrimDoubleQuotes()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(false, csvFileFormatProperties.isTrimDoubleQuotes()); } @Test @@ -134,7 +134,7 @@ public void testAnalyzeFileFormatPropertiesInvalidCompressType() { properties.put(CsvProperties.PROP_COMPRESS_TYPE, "invalid"); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); }); } @@ -143,15 +143,15 @@ public void testAnalyzeFileFormatPropertiesValidCompressType() throws AnalysisEx Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_COMPRESS_TYPE, "gz"); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(TFileCompressType.GZ, csvConfigurator.getCompressionType()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(TFileCompressType.GZ, csvFileFormatProperties.getCompressionType()); } @Test public void testAnalyzeFileFormatPropertiesEmptyCsvSchema() { Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_CSV_SCHEMA, ""); - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); } @Test @@ -160,7 +160,7 @@ public void testAnalyzeFileFormatPropertiesValidEncloseMultipleCharacters() { properties.put(CsvProperties.PROP_ENCLOSE, "\"\""); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); }); } @@ -169,8 +169,8 @@ public void testAnalyzeFileFormatPropertiesValidEncloseEmpty() { Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_ENCLOSE, ""); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(0, csvConfigurator.getEnclose()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(0, csvFileFormatProperties.getEnclose()); } @Test @@ -179,7 +179,7 @@ public void testAnalyzeFileFormatPropertiesSkipLinesAsString() { properties.put(CsvProperties.PROP_SKIP_LINES, "abc"); Assert.assertThrows(NumberFormatException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); }); } @@ -188,15 +188,15 @@ public void testAnalyzeFileFormatPropertiesValidColumnSeparator() throws Analysi Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_COLUMN_SEPARATOR, ";"); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(";", csvConfigurator.getColumnSeparator()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(";", csvFileFormatProperties.getColumnSeparator()); } @Test public void testAnalyzeFileFormatPropertiesLineDelimiterAsString() { Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_LINE_DELIMITER, "abc"); - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); } @Test @@ -204,8 +204,8 @@ public void testAnalyzeFileFormatPropertiesValidLineDelimiter() throws AnalysisE Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_LINE_DELIMITER, "\r\n"); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals("\r\n", csvConfigurator.getLineDelimiter()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals("\r\n", csvFileFormatProperties.getLineDelimiter()); } @Test @@ -213,8 +213,8 @@ public void testAnalyzeFileFormatPropertiesValidTrimDoubleQuotes() throws Analys Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(true, csvConfigurator.isTrimDoubleQuotes()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(true, csvFileFormatProperties.isTrimDoubleQuotes()); } @Test @@ -222,8 +222,8 @@ public void testAnalyzeFileFormatPropertiesInvalidTrimDoubleQuotes() { Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "invalid"); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(false, csvConfigurator.isTrimDoubleQuotes()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(false, csvFileFormatProperties.isTrimDoubleQuotes()); } @Test @@ -232,11 +232,11 @@ public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpaces() throws Ana properties.put(CsvProperties.PROP_CSV_SCHEMA, " column1:int ; column2:string ; column3:double "); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); - Assert.assertEquals("column1", csvConfigurator.getCsvSchema().get(0).getName().trim()); - Assert.assertEquals("column2", csvConfigurator.getCsvSchema().get(1).getName().trim()); - Assert.assertEquals("column3", csvConfigurator.getCsvSchema().get(2).getName().trim()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(3, csvFileFormatProperties.getCsvSchema().size()); + Assert.assertEquals("column1", csvFileFormatProperties.getCsvSchema().get(0).getName().trim()); + Assert.assertEquals("column2", csvFileFormatProperties.getCsvSchema().get(1).getName().trim()); + Assert.assertEquals("column3", csvFileFormatProperties.getCsvSchema().get(2).getName().trim()); } @Test @@ -245,11 +245,11 @@ public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpecialCharacters() properties.put(CsvProperties.PROP_CSV_SCHEMA, "col1@#$:int;col2&*:string;col3:double"); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); - Assert.assertEquals("col1@#$", csvConfigurator.getCsvSchema().get(0).getName()); - Assert.assertEquals("col2&*", csvConfigurator.getCsvSchema().get(1).getName()); - Assert.assertEquals("col3", csvConfigurator.getCsvSchema().get(2).getName()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(3, csvFileFormatProperties.getCsvSchema().size()); + Assert.assertEquals("col1@#$", csvFileFormatProperties.getCsvSchema().get(0).getName()); + Assert.assertEquals("col2&*", csvFileFormatProperties.getCsvSchema().get(1).getName()); + Assert.assertEquals("col3", csvFileFormatProperties.getCsvSchema().get(2).getName()); } @Test @@ -258,7 +258,7 @@ public void testAnalyzeFileFormatPropertiesInvalidCsvSchema() throws AnalysisExc properties.put(CsvProperties.PROP_CSV_SCHEMA, "column1,column2,column3"); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); }); } @@ -269,7 +269,7 @@ public void testAnalyzeFileFormatPropertiesInvalidCsvSchemaWithSpecialCharacters "col1@#$:int;col2&*():string;col3:double"); Assert.assertThrows(AnalysisException.class, () -> { - csvConfigurator.analyzeFileFormatProperties(properties, true); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); }); } @@ -278,7 +278,7 @@ public void testAnalyzeFileFormatPropertiesValidCsvSchema() throws AnalysisExcep Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_CSV_SCHEMA, "column1:int;column2:string;column3:double"); - csvConfigurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(3, csvConfigurator.getCsvSchema().size()); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(3, csvFileFormatProperties.getCsvSchema().size()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatPropertiesTest.java index ca9e9546fea7a9..74d8d0db2ad19b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/FileFormatPropertiesTest.java @@ -26,9 +26,9 @@ public class FileFormatPropertiesTest { @Test - public void testCreateFileFormatCheckerInvalidFormat() { + public void testCreateFileFormatPropertiesInvalidFormat() { Assert.assertThrows(AnalysisException.class, () -> { - FileFormatProperties.createFileFormatChecker("invalid_format"); + FileFormatProperties.createFileFormatProperties("invalid_format"); }); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java index d7e0be0ed57ba1..1d2499716166b2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java @@ -30,24 +30,24 @@ public class JsonFileFormatPropertiesTest { - private JsonFileFormatProperties configurator; + private JsonFileFormatProperties jsonFileFormatProperties; @Before public void setUp() { - configurator = new JsonFileFormatProperties(TFileFormatType.FORMAT_JSON); + jsonFileFormatProperties = new JsonFileFormatProperties(TFileFormatType.FORMAT_JSON); } @Test public void testAnalyzeFileFormatPropertiesEmpty() throws AnalysisException { Map properties = new HashMap<>(); - configurator.analyzeFileFormatProperties(properties, true); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); - Assert.assertEquals("", configurator.getJsonRoot()); - Assert.assertEquals("", configurator.getJsonPaths()); - Assert.assertEquals(false, configurator.isStripOuterArray()); - Assert.assertEquals(false, configurator.isReadJsonByLine()); - Assert.assertEquals(false, configurator.isNumAsString()); - Assert.assertEquals(false, configurator.isFuzzyParse()); + Assert.assertEquals("", jsonFileFormatProperties.getJsonRoot()); + Assert.assertEquals("", jsonFileFormatProperties.getJsonPaths()); + Assert.assertEquals(false, jsonFileFormatProperties.isStripOuterArray()); + Assert.assertEquals(false, jsonFileFormatProperties.isReadJsonByLine()); + Assert.assertEquals(false, jsonFileFormatProperties.isNumAsString()); + Assert.assertEquals(false, jsonFileFormatProperties.isFuzzyParse()); } @Test @@ -55,8 +55,8 @@ public void testAnalyzeFileFormatPropertiesValidJsonRoot() throws AnalysisExcept Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_JSON_ROOT, "data.items"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals("data.items", configurator.getJsonRoot()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals("data.items", jsonFileFormatProperties.getJsonRoot()); } @Test @@ -65,8 +65,8 @@ public void testAnalyzeFileFormatPropertiesValidJsonPaths() throws AnalysisExcep properties.put(JsonProperties.PROP_JSON_PATHS, "[\"$.name\", \"$.age\", \"$.city\"]"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals("[\"$.name\", \"$.age\", \"$.city\"]", configurator.getJsonPaths()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals("[\"$.name\", \"$.age\", \"$.city\"]", jsonFileFormatProperties.getJsonPaths()); } @Test @@ -74,8 +74,8 @@ public void testAnalyzeFileFormatPropertiesStripOuterArrayTrue() throws Analysis Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_STRIP_OUTER_ARRAY, "true"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(true, configurator.isStripOuterArray()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(true, jsonFileFormatProperties.isStripOuterArray()); } @Test @@ -83,8 +83,8 @@ public void testAnalyzeFileFormatPropertiesStripOuterArrayFalse() throws Analysi Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_STRIP_OUTER_ARRAY, "false"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(false, configurator.isStripOuterArray()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(false, jsonFileFormatProperties.isStripOuterArray()); } @Test @@ -92,8 +92,8 @@ public void testAnalyzeFileFormatPropertiesReadJsonByLineTrue() throws AnalysisE Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_READ_JSON_BY_LINE, "true"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(true, configurator.isReadJsonByLine()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(true, jsonFileFormatProperties.isReadJsonByLine()); } @Test @@ -101,8 +101,8 @@ public void testAnalyzeFileFormatPropertiesReadJsonByLineFalse() throws Analysis Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_READ_JSON_BY_LINE, "false"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(false, configurator.isReadJsonByLine()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(false, jsonFileFormatProperties.isReadJsonByLine()); } @Test @@ -110,8 +110,8 @@ public void testAnalyzeFileFormatPropertiesNumAsStringTrue() throws AnalysisExce Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_NUM_AS_STRING, "true"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(true, configurator.isNumAsString()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(true, jsonFileFormatProperties.isNumAsString()); } @Test @@ -119,8 +119,8 @@ public void testAnalyzeFileFormatPropertiesNumAsStringFalse() throws AnalysisExc Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_NUM_AS_STRING, "false"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(false, configurator.isNumAsString()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(false, jsonFileFormatProperties.isNumAsString()); } @Test @@ -128,8 +128,8 @@ public void testAnalyzeFileFormatPropertiesFuzzyParseTrue() throws AnalysisExcep Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_FUZZY_PARSE, "true"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(true, configurator.isFuzzyParse()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(true, jsonFileFormatProperties.isFuzzyParse()); } @Test @@ -137,8 +137,8 @@ public void testAnalyzeFileFormatPropertiesFuzzyParseFalse() throws AnalysisExce Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_FUZZY_PARSE, "false"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(false, configurator.isFuzzyParse()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(false, jsonFileFormatProperties.isFuzzyParse()); } @Test @@ -146,8 +146,8 @@ public void testAnalyzeFileFormatPropertiesInvalidBooleanValue() throws Analysis Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_FUZZY_PARSE, "invalid"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(false, configurator.isFuzzyParse()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(false, jsonFileFormatProperties.isFuzzyParse()); } @Test @@ -160,14 +160,14 @@ public void testAnalyzeFileFormatPropertiesAllProperties() throws AnalysisExcept properties.put(JsonProperties.PROP_NUM_AS_STRING, "true"); properties.put(JsonProperties.PROP_FUZZY_PARSE, "true"); - configurator.analyzeFileFormatProperties(properties, true); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); - Assert.assertEquals("data.records", configurator.getJsonRoot()); - Assert.assertEquals("[\"$.id\", \"$.name\"]", configurator.getJsonPaths()); - Assert.assertEquals(true, configurator.isStripOuterArray()); - Assert.assertEquals(true, configurator.isReadJsonByLine()); - Assert.assertEquals(true, configurator.isNumAsString()); - Assert.assertEquals(true, configurator.isFuzzyParse()); + Assert.assertEquals("data.records", jsonFileFormatProperties.getJsonRoot()); + Assert.assertEquals("[\"$.id\", \"$.name\"]", jsonFileFormatProperties.getJsonPaths()); + Assert.assertEquals(true, jsonFileFormatProperties.isStripOuterArray()); + Assert.assertEquals(true, jsonFileFormatProperties.isReadJsonByLine()); + Assert.assertEquals(true, jsonFileFormatProperties.isNumAsString()); + Assert.assertEquals(true, jsonFileFormatProperties.isFuzzyParse()); } @Test @@ -175,8 +175,8 @@ public void testAnalyzeFileFormatPropertiesSpecialCharactersInJsonRoot() throws Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_JSON_ROOT, "data.special@#$%^&*()"); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals("data.special@#$%^&*()", configurator.getJsonRoot()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals("data.special@#$%^&*()", jsonFileFormatProperties.getJsonRoot()); } @Test @@ -185,9 +185,9 @@ public void testAnalyzeFileFormatPropertiesComplexJsonPaths() throws AnalysisExc properties.put(JsonProperties.PROP_JSON_PATHS, "[\"$.deeply.nested[0].array[*].field\", \"$.complex.path[?(@.type=='value')]\"]"); - configurator.analyzeFileFormatProperties(properties, true); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals("[\"$.deeply.nested[0].array[*].field\", \"$.complex.path[?(@.type=='value')]\"]", - configurator.getJsonPaths()); + jsonFileFormatProperties.getJsonPaths()); } @Test @@ -195,7 +195,7 @@ public void testAnalyzeFileFormatPropertiesEmptyJsonPaths() throws AnalysisExcep Map properties = new HashMap<>(); properties.put(JsonProperties.PROP_JSON_PATHS, ""); - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals("", configurator.getJsonPaths()); + jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals("", jsonFileFormatProperties.getJsonPaths()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java index 8769c51217ef29..99dcdf427d1708 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java @@ -29,18 +29,18 @@ public class OrcFileFormatPropertiesTest { - private OrcFileFormatProperties configurator; + private OrcFileFormatProperties orcFileFormatProperties; @Before public void setUp() { - configurator = new OrcFileFormatProperties(TFileFormatType.FORMAT_ORC); + orcFileFormatProperties = new OrcFileFormatProperties(TFileFormatType.FORMAT_ORC); } @Test public void testAnalyzeFileFormatProperties() { Map properties = new HashMap<>(); // Add properties if needed - configurator.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(TFileCompressType.ZLIB, configurator.getOrcCompressionType()); + orcFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(TFileCompressType.ZLIB, orcFileFormatProperties.getOrcCompressionType()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java index b028b19e72dac1..faf172b9b98f50 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java @@ -29,20 +29,20 @@ public class ParquetFileFormatPropertiesTest { - private ParquetFileFormatProperties configurator; + private ParquetFileFormatProperties parquetFileFormatProperties; @Before public void setUp() { - configurator = new ParquetFileFormatProperties(TFileFormatType.FORMAT_PARQUET); + parquetFileFormatProperties = new ParquetFileFormatProperties(TFileFormatType.FORMAT_PARQUET); } @Test public void testAnalyzeFileFormatProperties() { Map properties = new HashMap<>(); // Add properties if needed - configurator.analyzeFileFormatProperties(properties, true); + parquetFileFormatProperties.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(TParquetCompressionType.SNAPPY, configurator.getParquetCompressionType()); - Assert.assertEquals(false, configurator.isParquetDisableDictionary()); + Assert.assertEquals(TParquetCompressionType.SNAPPY, parquetFileFormatProperties.getParquetCompressionType()); + Assert.assertEquals(false, parquetFileFormatProperties.isParquetDisableDictionary()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java index c7f269911941d2..f4ec8e90df2d32 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java @@ -27,17 +27,17 @@ public class WalFileFormatPropertiesTest { - private WalFileFormatProperties configurator; + private WalFileFormatProperties walFileFormatProperties; @Before public void setUp() { - configurator = new WalFileFormatProperties(TFileFormatType.FORMAT_WAL); + walFileFormatProperties = new WalFileFormatProperties(TFileFormatType.FORMAT_WAL); } @Test public void testAnalyzeFileFormatProperties() { Map properties = new HashMap<>(); // Add properties if needed - configurator.analyzeFileFormatProperties(properties, true); + walFileFormatProperties.analyzeFileFormatProperties(properties, true); } } From d409a9f5a667ebb42e4a4f57307c9234e0348a9d Mon Sep 17 00:00:00 2001 From: BePPPower Date: Thu, 24 Apr 2025 19:35:16 +0800 Subject: [PATCH 5/7] fix 5 --- .../org/apache/doris/common/util/Util.java | 6 ++- .../fileformat/AvroFileFormatProperties.java | 10 +---- .../fileformat/CsvFileFormatProperties.java | 45 ++++++------------- .../fileformat/FileFormatProperties.java | 39 ++++++++-------- .../fileformat/JsonFileFormatProperties.java | 31 +++++-------- .../fileformat/OrcFileFormatProperties.java | 10 +---- .../ParquetFileFormatProperties.java | 10 +---- .../fileformat/WalFileFormatProperties.java | 10 +---- .../AvroFileFormatPropertiesTest.java | 4 +- .../CsvFileFormatPropertiesTest.java | 3 +- .../JsonFileFormatPropertiesTest.java | 3 +- .../OrcFileFormatPropertiesTest.java | 3 +- .../ParquetFileFormatPropertiesTest.java | 3 +- .../WalFileFormatPropertiesTest.java | 4 +- 14 files changed, 61 insertions(+), 120 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index 8e808e3c097402..87d8e7332b6cdc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -614,7 +614,11 @@ public static TFileCompressType getFileCompressType(String compressType) { return TFileCompressType.UNKNOWN; } final String upperCaseType = compressType.toUpperCase(); - return TFileCompressType.valueOf(upperCaseType); + try { + return TFileCompressType.valueOf(upperCaseType); + } catch (IllegalArgumentException e) { + return TFileCompressType.UNKNOWN; + } } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java index 19c77b33646e57..f5a5c34a3f4b6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.property.fileformat; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileTextScanRangeParams; @@ -27,8 +26,8 @@ import java.util.Map; public class AvroFileFormatProperties extends FileFormatProperties { - public AvroFileFormatProperties(TFileFormatType fileFormatType) { - super(fileFormatType); + public AvroFileFormatProperties() { + super(TFileFormatType.FORMAT_AVRO); } @Override @@ -36,11 +35,6 @@ public void analyzeFileFormatProperties(Map formatProperties, bo throws AnalysisException { } - @Override - public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { - return null; - } - @Override public TResultFileSinkOptions toTResultFileSinkOptions() { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java index cd4479f57abe90..37928a16a98200 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java @@ -19,11 +19,9 @@ import org.apache.doris.analysis.Separator; import org.apache.doris.catalog.Column; -import org.apache.doris.common.util.FileFormatUtils; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.property.constants.CsvProperties; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileFormatType; @@ -57,19 +55,18 @@ public class CsvFileFormatProperties extends FileFormatProperties { String defaultColumnSeparator = CsvProperties.DEFAULT_COLUMN_SEPARATOR; - public CsvFileFormatProperties(TFileFormatType fileFormatType) { - super(fileFormatType); + public CsvFileFormatProperties() { + super(TFileFormatType.FORMAT_CSV_PLAIN); } - public CsvFileFormatProperties(TFileFormatType fileFormatType, String defaultColumnSeparator, - TTextSerdeType textSerdeType) { - super(fileFormatType); + public CsvFileFormatProperties(String defaultColumnSeparator, TTextSerdeType textSerdeType) { + super(TFileFormatType.FORMAT_CSV_PLAIN); this.defaultColumnSeparator = defaultColumnSeparator; this.textSerdeType = textSerdeType; } - public CsvFileFormatProperties(TFileFormatType fileFormatType, String headerType) { - super(fileFormatType); + public CsvFileFormatProperties(String headerType) { + super(TFileFormatType.FORMAT_CSV_PLAIN); this.headerType = headerType; } @@ -78,22 +75,22 @@ public CsvFileFormatProperties(TFileFormatType fileFormatType, String headerType public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException { try { - // check properties specified by user -- formatProperties - columnSeparator = getOrDefaultAndRemove(formatProperties, CsvProperties.PROP_COLUMN_SEPARATOR, + // analyze properties specified by user + columnSeparator = getOrDefault(formatProperties, CsvProperties.PROP_COLUMN_SEPARATOR, defaultColumnSeparator, isRemoveOriginProperty); if (Strings.isNullOrEmpty(columnSeparator)) { throw new AnalysisException("column_separator can not be empty."); } columnSeparator = Separator.convertSeparator(columnSeparator); - lineDelimiter = getOrDefaultAndRemove(formatProperties, CsvProperties.PROP_LINE_DELIMITER, + lineDelimiter = getOrDefault(formatProperties, CsvProperties.PROP_LINE_DELIMITER, CsvProperties.DEFAULT_LINE_DELIMITER, isRemoveOriginProperty); if (Strings.isNullOrEmpty(lineDelimiter)) { throw new AnalysisException("line_delimiter can not be empty."); } lineDelimiter = Separator.convertSeparator(lineDelimiter); - String enclosedString = getOrDefaultAndRemove(formatProperties, CsvProperties.PROP_ENCLOSE, + String enclosedString = getOrDefault(formatProperties, CsvProperties.PROP_ENCLOSE, "", isRemoveOriginProperty); if (!Strings.isNullOrEmpty(enclosedString)) { if (enclosedString.length() > 1) { @@ -105,38 +102,24 @@ public void analyzeFileFormatProperties(Map formatProperties, bo } } - trimDoubleQuotes = Boolean.valueOf(getOrDefaultAndRemove(formatProperties, + trimDoubleQuotes = Boolean.valueOf(getOrDefault(formatProperties, CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "", isRemoveOriginProperty)) .booleanValue(); - skipLines = Integer.valueOf(getOrDefaultAndRemove(formatProperties, + skipLines = Integer.valueOf(getOrDefault(formatProperties, CsvProperties.PROP_SKIP_LINES, "0", isRemoveOriginProperty)).intValue(); if (skipLines < 0) { throw new AnalysisException("skipLines should not be less than 0."); } - String compressTypeStr = getOrDefaultAndRemove(formatProperties, + String compressTypeStr = getOrDefault(formatProperties, CsvProperties.PROP_COMPRESS_TYPE, "UNKNOWN", isRemoveOriginProperty); - try { - compressionType = Util.getFileCompressType(compressTypeStr); - } catch (IllegalArgumentException e) { - throw new AnalysisException("Compress type : " + compressTypeStr + " is not supported."); - } + compressionType = Util.getFileCompressType(compressTypeStr); - FileFormatUtils.parseCsvSchema(csvSchema, getOrDefaultAndRemove(formatProperties, - CsvProperties.PROP_CSV_SCHEMA, "", isRemoveOriginProperty)); - if (LOG.isDebugEnabled()) { - LOG.debug("get csv schema: {}", csvSchema); - } } catch (org.apache.doris.common.AnalysisException e) { throw new AnalysisException(e.getMessage()); } } - @Override - public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { - return null; - } - @Override public TResultFileSinkOptions toTResultFileSinkOptions() { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java index e5cd97613e070a..bb1fdc0d72d925 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java @@ -20,7 +20,6 @@ import org.apache.doris.datasource.property.constants.CsvProperties; import org.apache.doris.datasource.property.constants.FileFormatBaseProperties; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TFileFormatType; @@ -38,13 +37,6 @@ public FileFormatProperties(TFileFormatType fileFormatType) { this.fileFormatType = fileFormatType; } - /** - * - * @param formatProperties - * @return properties needed by Doris - * @throws AnalysisException - */ - /** * Analyze user properties * @param formatProperties properties specified by user @@ -55,36 +47,41 @@ public abstract void analyzeFileFormatProperties( Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException; - public abstract PFetchTableSchemaRequest toPFetchTableSchemaRequest(); - + /** + * generate TResultFileSinkOptions according to the properties of specified file format + * You must call method `analyzeFileFormatProperties` once before calling method `toTResultFileSinkOptions` + */ public abstract TResultFileSinkOptions toTResultFileSinkOptions(); + /** + * generate TFileAttributes according to the properties of specified file format + * You must call method `analyzeFileFormatProperties` once before calling method `toTFileAttributes` + */ public abstract TFileAttributes toTFileAttributes(); public static FileFormatProperties createFileFormatProperties(String formatString) { switch (formatString) { case FileFormatBaseProperties.FORMAT_CSV: - return new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN); + return new CsvFileFormatProperties(); case FileFormatBaseProperties.FORMAT_HIVE_TEXT: - return new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN, - CsvProperties.DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR, + return new CsvFileFormatProperties(CsvProperties.DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR, TTextSerdeType.HIVE_TEXT_SERDE); case FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES: - return new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN, + return new CsvFileFormatProperties( FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES); case FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES_AND_TYPES: - return new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN, + return new CsvFileFormatProperties( FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES_AND_TYPES); case FileFormatBaseProperties.FORMAT_PARQUET: - return new ParquetFileFormatProperties(TFileFormatType.FORMAT_PARQUET); + return new ParquetFileFormatProperties(); case FileFormatBaseProperties.FORMAT_ORC: - return new OrcFileFormatProperties(TFileFormatType.FORMAT_ORC); + return new OrcFileFormatProperties(); case FileFormatBaseProperties.FORMAT_JSON: - return new JsonFileFormatProperties(TFileFormatType.FORMAT_JSON); + return new JsonFileFormatProperties(); case FileFormatBaseProperties.FORMAT_AVRO: - return new AvroFileFormatProperties(TFileFormatType.FORMAT_AVRO); + return new AvroFileFormatProperties(); case FileFormatBaseProperties.FORMAT_WAL: - return new WalFileFormatProperties(TFileFormatType.FORMAT_WAL); + return new WalFileFormatProperties(); default: throw new AnalysisException("format:" + formatString + " is not supported."); } @@ -97,7 +94,7 @@ public static FileFormatProperties createFileFormatProperties(Map props, String key, String defaultValue, + protected String getOrDefault(Map props, String key, String defaultValue, boolean isRemove) { String value = props.getOrDefault(key, defaultValue); if (isRemove) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java index 65cacf0c41f1c5..6f15492d573532 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java @@ -20,7 +20,6 @@ import org.apache.doris.common.util.FileFormatConstants; import org.apache.doris.common.util.Util; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileTextScanRangeParams; @@ -38,43 +37,33 @@ public class JsonFileFormatProperties extends FileFormatProperties { private boolean fuzzyParse; - public JsonFileFormatProperties(TFileFormatType fileFormatType) { - super(fileFormatType); + public JsonFileFormatProperties() { + super(TFileFormatType.FORMAT_JSON); } @Override public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException { - // 这几个json应该移到json checker中 - jsonRoot = getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_JSON_ROOT, + jsonRoot = getOrDefault(formatProperties, FileFormatConstants.PROP_JSON_ROOT, "", isRemoveOriginProperty); - jsonPaths = getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_JSON_PATHS, + jsonPaths = getOrDefault(formatProperties, FileFormatConstants.PROP_JSON_PATHS, "", isRemoveOriginProperty); readJsonByLine = Boolean.valueOf( - getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_READ_JSON_BY_LINE, + getOrDefault(formatProperties, FileFormatConstants.PROP_READ_JSON_BY_LINE, "", isRemoveOriginProperty)).booleanValue(); stripOuterArray = Boolean.valueOf( - getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_STRIP_OUTER_ARRAY, + getOrDefault(formatProperties, FileFormatConstants.PROP_STRIP_OUTER_ARRAY, "", isRemoveOriginProperty)).booleanValue(); numAsString = Boolean.valueOf( - getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_NUM_AS_STRING, + getOrDefault(formatProperties, FileFormatConstants.PROP_NUM_AS_STRING, "", isRemoveOriginProperty)).booleanValue(); fuzzyParse = Boolean.valueOf( - getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_FUZZY_PARSE, + getOrDefault(formatProperties, FileFormatConstants.PROP_FUZZY_PARSE, "", isRemoveOriginProperty)).booleanValue(); - String compressTypeStr = getOrDefaultAndRemove(formatProperties, FileFormatConstants.PROP_COMPRESS_TYPE, + String compressTypeStr = getOrDefault(formatProperties, FileFormatConstants.PROP_COMPRESS_TYPE, "UNKNOWN", isRemoveOriginProperty); - try { - compressionType = Util.getFileCompressType(compressTypeStr); - } catch (IllegalArgumentException e) { - throw new AnalysisException("Compress type : " + compressTypeStr + " is not supported."); - } - } - - @Override - public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { - return null; + compressionType = Util.getFileCompressType(compressTypeStr); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatProperties.java index 6deb5b22bf78d6..68081c275e4aee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatProperties.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.property.fileformat; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TFileFormatType; @@ -30,8 +29,8 @@ public class OrcFileFormatProperties extends FileFormatProperties { private TFileCompressType orcCompressionType = TFileCompressType.ZLIB; - public OrcFileFormatProperties(TFileFormatType fileFormatType) { - super(fileFormatType); + public OrcFileFormatProperties() { + super(TFileFormatType.FORMAT_ORC); } @Override @@ -39,11 +38,6 @@ public void analyzeFileFormatProperties(Map formatProperties, bo throws AnalysisException { } - @Override - public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { - return null; - } - @Override public TResultFileSinkOptions toTResultFileSinkOptions() { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java index e4b9dfe78f32ab..14fac8fc9bce16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.property.fileformat; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileTextScanRangeParams; @@ -31,8 +30,8 @@ public class ParquetFileFormatProperties extends FileFormatProperties { private TParquetCompressionType parquetCompressionType = TParquetCompressionType.SNAPPY; private boolean parquetDisableDictionary = false; - public ParquetFileFormatProperties(TFileFormatType fileFormatType) { - super(fileFormatType); + public ParquetFileFormatProperties() { + super(TFileFormatType.FORMAT_PARQUET); } @Override @@ -40,11 +39,6 @@ public void analyzeFileFormatProperties(Map formatProperties, bo throws AnalysisException { } - @Override - public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { - return null; - } - @Override public TResultFileSinkOptions toTResultFileSinkOptions() { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatProperties.java index f935fa38b34a40..ba37bcc8534ac4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/WalFileFormatProperties.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.property.fileformat; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileTextScanRangeParams; @@ -27,13 +26,8 @@ import java.util.Map; public class WalFileFormatProperties extends FileFormatProperties { - public WalFileFormatProperties(TFileFormatType fileFormatType) { - super(fileFormatType); - } - - @Override - public PFetchTableSchemaRequest toPFetchTableSchemaRequest() { - return null; + public WalFileFormatProperties() { + super(TFileFormatType.FORMAT_WAL); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java index b1885a1e46ce8b..a7fc534e0de5cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java @@ -17,8 +17,6 @@ package org.apache.doris.datasource.property.fileformat; -import org.apache.doris.thrift.TFileFormatType; - import org.junit.Before; import org.junit.Test; @@ -32,7 +30,7 @@ public class AvroFileFormatPropertiesTest { @Before public void setUp() { - avroFileFormatProperties = new AvroFileFormatProperties(TFileFormatType.FORMAT_AVRO); + avroFileFormatProperties = new AvroFileFormatProperties(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java index 34654c18718c46..966bf372f40003 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java @@ -20,7 +20,6 @@ import org.apache.doris.datasource.property.constants.CsvProperties; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TFileFormatType; import org.junit.Assert; import org.junit.Before; @@ -35,7 +34,7 @@ public class CsvFileFormatPropertiesTest { @Before public void setUp() { - csvFileFormatProperties = new CsvFileFormatProperties(TFileFormatType.FORMAT_CSV_PLAIN); + csvFileFormatProperties = new CsvFileFormatProperties(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java index 1d2499716166b2..3e388660dde57d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java @@ -19,7 +19,6 @@ import org.apache.doris.datasource.property.constants.JsonProperties; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.thrift.TFileFormatType; import org.junit.Assert; import org.junit.Before; @@ -34,7 +33,7 @@ public class JsonFileFormatPropertiesTest { @Before public void setUp() { - jsonFileFormatProperties = new JsonFileFormatProperties(TFileFormatType.FORMAT_JSON); + jsonFileFormatProperties = new JsonFileFormatProperties(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java index 99dcdf427d1708..2db57de674c42c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/OrcFileFormatPropertiesTest.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.property.fileformat; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TFileFormatType; import org.junit.Assert; import org.junit.Before; @@ -33,7 +32,7 @@ public class OrcFileFormatPropertiesTest { @Before public void setUp() { - orcFileFormatProperties = new OrcFileFormatProperties(TFileFormatType.FORMAT_ORC); + orcFileFormatProperties = new OrcFileFormatProperties(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java index faf172b9b98f50..17b99dd7065c91 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatPropertiesTest.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.property.fileformat; -import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TParquetCompressionType; import org.junit.Assert; @@ -33,7 +32,7 @@ public class ParquetFileFormatPropertiesTest { @Before public void setUp() { - parquetFileFormatProperties = new ParquetFileFormatProperties(TFileFormatType.FORMAT_PARQUET); + parquetFileFormatProperties = new ParquetFileFormatProperties(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java index f4ec8e90df2d32..d94b49aca978f4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/WalFileFormatPropertiesTest.java @@ -17,8 +17,6 @@ package org.apache.doris.datasource.property.fileformat; -import org.apache.doris.thrift.TFileFormatType; - import org.junit.Before; import org.junit.Test; @@ -31,7 +29,7 @@ public class WalFileFormatPropertiesTest { @Before public void setUp() { - walFileFormatProperties = new WalFileFormatProperties(TFileFormatType.FORMAT_WAL); + walFileFormatProperties = new WalFileFormatProperties(); } @Test From 8400f1d1246939caf71e919e59135bed860a573d Mon Sep 17 00:00:00 2001 From: BePPPower Date: Thu, 24 Apr 2025 21:59:10 +0800 Subject: [PATCH 6/7] fix 6 --- .../CsvFileFormatPropertiesTest.java | 62 +------------------ 1 file changed, 2 insertions(+), 60 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java index 966bf372f40003..7c7fc8b10abb15 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java @@ -131,10 +131,8 @@ public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesFalse() throws Analys public void testAnalyzeFileFormatPropertiesInvalidCompressType() { Map properties = new HashMap<>(); properties.put(CsvProperties.PROP_COMPRESS_TYPE, "invalid"); - - Assert.assertThrows(AnalysisException.class, () -> { - csvFileFormatProperties.analyzeFileFormatProperties(properties, true); - }); + csvFileFormatProperties.analyzeFileFormatProperties(properties, true); + Assert.assertEquals(TFileCompressType.UNKNOWN, csvFileFormatProperties.getCompressionType()); } @Test @@ -224,60 +222,4 @@ public void testAnalyzeFileFormatPropertiesInvalidTrimDoubleQuotes() { csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, csvFileFormatProperties.isTrimDoubleQuotes()); } - - @Test - public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpaces() throws AnalysisException { - Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_CSV_SCHEMA, - " column1:int ; column2:string ; column3:double "); - - csvFileFormatProperties.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(3, csvFileFormatProperties.getCsvSchema().size()); - Assert.assertEquals("column1", csvFileFormatProperties.getCsvSchema().get(0).getName().trim()); - Assert.assertEquals("column2", csvFileFormatProperties.getCsvSchema().get(1).getName().trim()); - Assert.assertEquals("column3", csvFileFormatProperties.getCsvSchema().get(2).getName().trim()); - } - - @Test - public void testAnalyzeFileFormatPropertiesValidCsvSchemaWithSpecialCharacters() throws AnalysisException { - Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_CSV_SCHEMA, - "col1@#$:int;col2&*:string;col3:double"); - - csvFileFormatProperties.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(3, csvFileFormatProperties.getCsvSchema().size()); - Assert.assertEquals("col1@#$", csvFileFormatProperties.getCsvSchema().get(0).getName()); - Assert.assertEquals("col2&*", csvFileFormatProperties.getCsvSchema().get(1).getName()); - Assert.assertEquals("col3", csvFileFormatProperties.getCsvSchema().get(2).getName()); - } - - @Test - public void testAnalyzeFileFormatPropertiesInvalidCsvSchema() throws AnalysisException { - Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_CSV_SCHEMA, "column1,column2,column3"); - - Assert.assertThrows(AnalysisException.class, () -> { - csvFileFormatProperties.analyzeFileFormatProperties(properties, true); - }); - } - - @Test - public void testAnalyzeFileFormatPropertiesInvalidCsvSchemaWithSpecialCharacters() throws AnalysisException { - Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_CSV_SCHEMA, - "col1@#$:int;col2&*():string;col3:double"); - - Assert.assertThrows(AnalysisException.class, () -> { - csvFileFormatProperties.analyzeFileFormatProperties(properties, true); - }); - } - - @Test - public void testAnalyzeFileFormatPropertiesValidCsvSchema() throws AnalysisException { - Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_CSV_SCHEMA, - "column1:int;column2:string;column3:double"); - csvFileFormatProperties.analyzeFileFormatProperties(properties, true); - Assert.assertEquals(3, csvFileFormatProperties.getCsvSchema().size()); - } } From cc56925cee961d439626ba77da726a23c69e2849 Mon Sep 17 00:00:00 2001 From: BePPPower Date: Fri, 25 Apr 2025 14:41:02 +0800 Subject: [PATCH 7/7] fix 7 --- .../property/constants/AvroProperties.java | 21 --------- .../property/constants/CsvProperties.java | 34 -------------- .../constants/FileFormatBaseProperties.java | 32 ------------- .../property/constants/JsonProperties.java | 27 ----------- .../property/constants/OrcProperties.java | 22 --------- .../property/constants/ParquetProperties.java | 26 ----------- .../property/constants/WalProperties.java | 21 --------- .../fileformat/CsvFileFormatProperties.java | 35 ++++++++++----- .../fileformat/FileFormatProperties.java | 41 ++++++++++------- .../fileformat/JsonFileFormatProperties.java | 22 +++++---- .../ParquetFileFormatProperties.java | 5 +++ .../CsvFileFormatPropertiesTest.java | 45 +++++++++---------- .../JsonFileFormatPropertiesTest.java | 41 +++++++++-------- 13 files changed, 111 insertions(+), 261 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AvroProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CsvProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/FileFormatBaseProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/JsonProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OrcProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ParquetProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/WalProperties.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AvroProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AvroProperties.java deleted file mode 100644 index 80846307ffe95d..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AvroProperties.java +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -public class AvroProperties { -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CsvProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CsvProperties.java deleted file mode 100644 index df54022402a417..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CsvProperties.java +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -public class CsvProperties { - public static final String DEFAULT_COLUMN_SEPARATOR = "\t"; - public static final String DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR = "\001"; - public static final String DEFAULT_LINE_DELIMITER = "\n"; - - public static final String PROP_COLUMN_SEPARATOR = "column_separator"; - public static final String PROP_LINE_DELIMITER = "line_delimiter"; - - public static final String PROP_SKIP_LINES = "skip_lines"; - public static final String PROP_CSV_SCHEMA = "csv_schema"; - public static final String PROP_COMPRESS_TYPE = "compress_type"; - public static final String PROP_TRIM_DOUBLE_QUOTES = "trim_double_quotes"; - - public static final String PROP_ENCLOSE = "enclose"; -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/FileFormatBaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/FileFormatBaseProperties.java deleted file mode 100644 index 19a29432732181..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/FileFormatBaseProperties.java +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -public class FileFormatBaseProperties { - public static final String PROP_FORMAT = "format"; - public static final String FORMAT_PARQUET = "parquet"; - public static final String FORMAT_CSV = "csv"; - public static final String FORMAT_CSV_WITH_NAMES = "csv_with_names"; - public static final String FORMAT_CSV_WITH_NAMES_AND_TYPES = "csv_with_names_and_types"; - public static final String FORMAT_HIVE_TEXT = "hive_text"; - public static final String FORMAT_ORC = "orc"; - public static final String FORMAT_JSON = "json"; - public static final String FORMAT_AVRO = "avro"; - public static final String FORMAT_WAL = "wal"; - public static final String FORMAT_ARROW = "arrow"; -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/JsonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/JsonProperties.java deleted file mode 100644 index 7b48a29a14a355..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/JsonProperties.java +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -public class JsonProperties { - public static final String PROP_JSON_ROOT = "json_root"; - public static final String PROP_JSON_PATHS = "jsonpaths"; - public static final String PROP_STRIP_OUTER_ARRAY = "strip_outer_array"; - public static final String PROP_READ_JSON_BY_LINE = "read_json_by_line"; - public static final String PROP_NUM_AS_STRING = "num_as_string"; - public static final String PROP_FUZZY_PARSE = "fuzzy_parse"; -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OrcProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OrcProperties.java deleted file mode 100644 index 797c71e76f0d69..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OrcProperties.java +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -public class OrcProperties { - public static final String COMPRESS_TYPE = "compress_type"; -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ParquetProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ParquetProperties.java deleted file mode 100644 index 6685566ac25cf8..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ParquetProperties.java +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.thrift.TParquetVersion; - -public class ParquetProperties { - public static final String PARQUET_DISABLE_DICTIONARY = "disable_dictionary"; - public static final TParquetVersion parquetVersion = TParquetVersion.PARQUET_1_0; - public static final String PARQUET_VERSION = "version"; -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/WalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/WalProperties.java deleted file mode 100644 index 9b535132fa624b..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/WalProperties.java +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -public class WalProperties { -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java index 37928a16a98200..74a9a58aec7a26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java @@ -20,7 +20,6 @@ import org.apache.doris.analysis.Separator; import org.apache.doris.catalog.Column; import org.apache.doris.common.util.Util; -import org.apache.doris.datasource.property.constants.CsvProperties; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileAttributes; @@ -41,10 +40,24 @@ public class CsvFileFormatProperties extends FileFormatProperties { public static final Logger LOG = LogManager.getLogger( org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties.class); + public static final String DEFAULT_COLUMN_SEPARATOR = "\t"; + public static final String DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR = "\001"; + public static final String DEFAULT_LINE_DELIMITER = "\n"; + + public static final String PROP_COLUMN_SEPARATOR = "column_separator"; + public static final String PROP_LINE_DELIMITER = "line_delimiter"; + + public static final String PROP_SKIP_LINES = "skip_lines"; + public static final String PROP_CSV_SCHEMA = "csv_schema"; + public static final String PROP_COMPRESS_TYPE = "compress_type"; + public static final String PROP_TRIM_DOUBLE_QUOTES = "trim_double_quotes"; + + public static final String PROP_ENCLOSE = "enclose"; + private String headerType = ""; private TTextSerdeType textSerdeType = TTextSerdeType.JSON_TEXT_SERDE; - private String columnSeparator = CsvProperties.DEFAULT_COLUMN_SEPARATOR; - private String lineDelimiter = CsvProperties.DEFAULT_LINE_DELIMITER; + private String columnSeparator = DEFAULT_COLUMN_SEPARATOR; + private String lineDelimiter = DEFAULT_LINE_DELIMITER; private boolean trimDoubleQuotes; private int skipLines; private byte enclose; @@ -53,7 +66,7 @@ public class CsvFileFormatProperties extends FileFormatProperties { // User specified csv columns, it will override columns got from file private final List csvSchema = Lists.newArrayList(); - String defaultColumnSeparator = CsvProperties.DEFAULT_COLUMN_SEPARATOR; + String defaultColumnSeparator = DEFAULT_COLUMN_SEPARATOR; public CsvFileFormatProperties() { super(TFileFormatType.FORMAT_CSV_PLAIN); @@ -76,21 +89,21 @@ public void analyzeFileFormatProperties(Map formatProperties, bo throws AnalysisException { try { // analyze properties specified by user - columnSeparator = getOrDefault(formatProperties, CsvProperties.PROP_COLUMN_SEPARATOR, + columnSeparator = getOrDefault(formatProperties, PROP_COLUMN_SEPARATOR, defaultColumnSeparator, isRemoveOriginProperty); if (Strings.isNullOrEmpty(columnSeparator)) { throw new AnalysisException("column_separator can not be empty."); } columnSeparator = Separator.convertSeparator(columnSeparator); - lineDelimiter = getOrDefault(formatProperties, CsvProperties.PROP_LINE_DELIMITER, - CsvProperties.DEFAULT_LINE_DELIMITER, isRemoveOriginProperty); + lineDelimiter = getOrDefault(formatProperties, PROP_LINE_DELIMITER, + DEFAULT_LINE_DELIMITER, isRemoveOriginProperty); if (Strings.isNullOrEmpty(lineDelimiter)) { throw new AnalysisException("line_delimiter can not be empty."); } lineDelimiter = Separator.convertSeparator(lineDelimiter); - String enclosedString = getOrDefault(formatProperties, CsvProperties.PROP_ENCLOSE, + String enclosedString = getOrDefault(formatProperties, PROP_ENCLOSE, "", isRemoveOriginProperty); if (!Strings.isNullOrEmpty(enclosedString)) { if (enclosedString.length() > 1) { @@ -103,16 +116,16 @@ public void analyzeFileFormatProperties(Map formatProperties, bo } trimDoubleQuotes = Boolean.valueOf(getOrDefault(formatProperties, - CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "", isRemoveOriginProperty)) + PROP_TRIM_DOUBLE_QUOTES, "", isRemoveOriginProperty)) .booleanValue(); skipLines = Integer.valueOf(getOrDefault(formatProperties, - CsvProperties.PROP_SKIP_LINES, "0", isRemoveOriginProperty)).intValue(); + PROP_SKIP_LINES, "0", isRemoveOriginProperty)).intValue(); if (skipLines < 0) { throw new AnalysisException("skipLines should not be less than 0."); } String compressTypeStr = getOrDefault(formatProperties, - CsvProperties.PROP_COMPRESS_TYPE, "UNKNOWN", isRemoveOriginProperty); + PROP_COMPRESS_TYPE, "UNKNOWN", isRemoveOriginProperty); compressionType = Util.getFileCompressType(compressTypeStr); } catch (org.apache.doris.common.AnalysisException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java index bb1fdc0d72d925..ad51b06ed425e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java @@ -17,8 +17,6 @@ package org.apache.doris.datasource.property.fileformat; -import org.apache.doris.datasource.property.constants.CsvProperties; -import org.apache.doris.datasource.property.constants.FileFormatBaseProperties; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileCompressType; @@ -29,6 +27,19 @@ import java.util.Map; public abstract class FileFormatProperties { + public static final String PROP_FORMAT = "format"; + public static final String FORMAT_PARQUET = "parquet"; + public static final String FORMAT_CSV = "csv"; + public static final String FORMAT_CSV_WITH_NAMES = "csv_with_names"; + public static final String FORMAT_CSV_WITH_NAMES_AND_TYPES = "csv_with_names_and_types"; + public static final String FORMAT_HIVE_TEXT = "hive_text"; + public static final String FORMAT_ORC = "orc"; + public static final String FORMAT_JSON = "json"; + public static final String FORMAT_AVRO = "avro"; + public static final String FORMAT_WAL = "wal"; + public static final String FORMAT_ARROW = "arrow"; + public static final String PROP_COMPRESS_TYPE = "compress_type"; + protected TFileFormatType fileFormatType; protected TFileCompressType compressionType; @@ -61,26 +72,26 @@ public abstract void analyzeFileFormatProperties( public static FileFormatProperties createFileFormatProperties(String formatString) { switch (formatString) { - case FileFormatBaseProperties.FORMAT_CSV: + case FORMAT_CSV: return new CsvFileFormatProperties(); - case FileFormatBaseProperties.FORMAT_HIVE_TEXT: - return new CsvFileFormatProperties(CsvProperties.DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR, + case FORMAT_HIVE_TEXT: + return new CsvFileFormatProperties(CsvFileFormatProperties.DEFAULT_HIVE_TEXT_COLUMN_SEPARATOR, TTextSerdeType.HIVE_TEXT_SERDE); - case FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES: + case FORMAT_CSV_WITH_NAMES: return new CsvFileFormatProperties( - FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES); - case FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES_AND_TYPES: + FORMAT_CSV_WITH_NAMES); + case FORMAT_CSV_WITH_NAMES_AND_TYPES: return new CsvFileFormatProperties( - FileFormatBaseProperties.FORMAT_CSV_WITH_NAMES_AND_TYPES); - case FileFormatBaseProperties.FORMAT_PARQUET: + FORMAT_CSV_WITH_NAMES_AND_TYPES); + case FORMAT_PARQUET: return new ParquetFileFormatProperties(); - case FileFormatBaseProperties.FORMAT_ORC: + case FORMAT_ORC: return new OrcFileFormatProperties(); - case FileFormatBaseProperties.FORMAT_JSON: + case FORMAT_JSON: return new JsonFileFormatProperties(); - case FileFormatBaseProperties.FORMAT_AVRO: + case FORMAT_AVRO: return new AvroFileFormatProperties(); - case FileFormatBaseProperties.FORMAT_WAL: + case FORMAT_WAL: return new WalFileFormatProperties(); default: throw new AnalysisException("format:" + formatString + " is not supported."); @@ -89,7 +100,7 @@ public static FileFormatProperties createFileFormatProperties(String formatStrin public static FileFormatProperties createFileFormatProperties(Map formatProperties) throws AnalysisException { - String formatString = formatProperties.getOrDefault(FileFormatBaseProperties.PROP_FORMAT, "") + String formatString = formatProperties.getOrDefault(PROP_FORMAT, "") .toLowerCase(); return createFileFormatProperties(formatString); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java index 6f15492d573532..4ed03f455d5505 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatProperties.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.property.fileformat; -import org.apache.doris.common.util.FileFormatConstants; import org.apache.doris.common.util.Util; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.thrift.TFileAttributes; @@ -28,6 +27,13 @@ import java.util.Map; public class JsonFileFormatProperties extends FileFormatProperties { + public static final String PROP_JSON_ROOT = "json_root"; + public static final String PROP_JSON_PATHS = "jsonpaths"; + public static final String PROP_STRIP_OUTER_ARRAY = "strip_outer_array"; + public static final String PROP_READ_JSON_BY_LINE = "read_json_by_line"; + public static final String PROP_NUM_AS_STRING = "num_as_string"; + public static final String PROP_FUZZY_PARSE = "fuzzy_parse"; + // from ExternalFileTableValuedFunction: private String jsonRoot = ""; private String jsonPaths = ""; @@ -44,24 +50,24 @@ public JsonFileFormatProperties() { @Override public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) throws AnalysisException { - jsonRoot = getOrDefault(formatProperties, FileFormatConstants.PROP_JSON_ROOT, + jsonRoot = getOrDefault(formatProperties, PROP_JSON_ROOT, "", isRemoveOriginProperty); - jsonPaths = getOrDefault(formatProperties, FileFormatConstants.PROP_JSON_PATHS, + jsonPaths = getOrDefault(formatProperties, PROP_JSON_PATHS, "", isRemoveOriginProperty); readJsonByLine = Boolean.valueOf( - getOrDefault(formatProperties, FileFormatConstants.PROP_READ_JSON_BY_LINE, + getOrDefault(formatProperties, PROP_READ_JSON_BY_LINE, "", isRemoveOriginProperty)).booleanValue(); stripOuterArray = Boolean.valueOf( - getOrDefault(formatProperties, FileFormatConstants.PROP_STRIP_OUTER_ARRAY, + getOrDefault(formatProperties, PROP_STRIP_OUTER_ARRAY, "", isRemoveOriginProperty)).booleanValue(); numAsString = Boolean.valueOf( - getOrDefault(formatProperties, FileFormatConstants.PROP_NUM_AS_STRING, + getOrDefault(formatProperties, PROP_NUM_AS_STRING, "", isRemoveOriginProperty)).booleanValue(); fuzzyParse = Boolean.valueOf( - getOrDefault(formatProperties, FileFormatConstants.PROP_FUZZY_PARSE, + getOrDefault(formatProperties, PROP_FUZZY_PARSE, "", isRemoveOriginProperty)).booleanValue(); - String compressTypeStr = getOrDefault(formatProperties, FileFormatConstants.PROP_COMPRESS_TYPE, + String compressTypeStr = getOrDefault(formatProperties, PROP_COMPRESS_TYPE, "UNKNOWN", isRemoveOriginProperty); compressionType = Util.getFileCompressType(compressTypeStr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java index 14fac8fc9bce16..565df5ba22219e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/ParquetFileFormatProperties.java @@ -22,11 +22,16 @@ import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileTextScanRangeParams; import org.apache.doris.thrift.TParquetCompressionType; +import org.apache.doris.thrift.TParquetVersion; import org.apache.doris.thrift.TResultFileSinkOptions; import java.util.Map; public class ParquetFileFormatProperties extends FileFormatProperties { + public static final String PARQUET_DISABLE_DICTIONARY = "disable_dictionary"; + public static final TParquetVersion parquetVersion = TParquetVersion.PARQUET_1_0; + public static final String PARQUET_VERSION = "version"; + private TParquetCompressionType parquetCompressionType = TParquetCompressionType.SNAPPY; private boolean parquetDisableDictionary = false; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java index 7c7fc8b10abb15..a496378b5e57ea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatPropertiesTest.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.property.fileformat; -import org.apache.doris.datasource.property.constants.CsvProperties; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.thrift.TFileCompressType; @@ -40,9 +39,9 @@ public void setUp() { @Test public void testAnalyzeFileFormatPropertiesValid() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_COLUMN_SEPARATOR, ","); - properties.put(CsvProperties.PROP_LINE_DELIMITER, "\n"); - properties.put(CsvProperties.PROP_SKIP_LINES, "1"); + properties.put(CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ","); + properties.put(CsvFileFormatProperties.PROP_LINE_DELIMITER, "\n"); + properties.put(CsvFileFormatProperties.PROP_SKIP_LINES, "1"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); @@ -54,7 +53,7 @@ public void testAnalyzeFileFormatPropertiesValid() throws AnalysisException { @Test public void testAnalyzeFileFormatPropertiesInvalidSeparator() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_COLUMN_SEPARATOR, ""); + properties.put(CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ""); Assert.assertThrows(AnalysisException.class, () -> { csvFileFormatProperties.analyzeFileFormatProperties(properties, true); @@ -64,7 +63,7 @@ public void testAnalyzeFileFormatPropertiesInvalidSeparator() { @Test public void testAnalyzeFileFormatPropertiesInvalidLineDelimiter() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_LINE_DELIMITER, ""); + properties.put(CsvFileFormatProperties.PROP_LINE_DELIMITER, ""); Assert.assertThrows(AnalysisException.class, () -> { csvFileFormatProperties.analyzeFileFormatProperties(properties, true); @@ -74,7 +73,7 @@ public void testAnalyzeFileFormatPropertiesInvalidLineDelimiter() { @Test public void testAnalyzeFileFormatPropertiesInvalidEnclose() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_ENCLOSE, "invalid"); + properties.put(CsvFileFormatProperties.PROP_ENCLOSE, "invalid"); Assert.assertThrows(AnalysisException.class, () -> { csvFileFormatProperties.analyzeFileFormatProperties(properties, true); @@ -84,7 +83,7 @@ public void testAnalyzeFileFormatPropertiesInvalidEnclose() { @Test public void testAnalyzeFileFormatPropertiesValidEnclose() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_ENCLOSE, "\""); + properties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals((byte) '"', csvFileFormatProperties.getEnclose()); @@ -93,7 +92,7 @@ public void testAnalyzeFileFormatPropertiesValidEnclose() throws AnalysisExcepti @Test public void testAnalyzeFileFormatPropertiesSkipLinesNegative() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_SKIP_LINES, "-1"); + properties.put(CsvFileFormatProperties.PROP_SKIP_LINES, "-1"); Assert.assertThrows(AnalysisException.class, () -> { csvFileFormatProperties.analyzeFileFormatProperties(properties, true); @@ -103,7 +102,7 @@ public void testAnalyzeFileFormatPropertiesSkipLinesNegative() { @Test public void testAnalyzeFileFormatPropertiesSkipLinesLargeValue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_SKIP_LINES, "1000"); + properties.put(CsvFileFormatProperties.PROP_SKIP_LINES, "1000"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(1000, csvFileFormatProperties.getSkipLines()); @@ -112,7 +111,7 @@ public void testAnalyzeFileFormatPropertiesSkipLinesLargeValue() throws Analysis @Test public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesTrue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); + properties.put(CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, csvFileFormatProperties.isTrimDoubleQuotes()); @@ -121,7 +120,7 @@ public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesTrue() throws Analysi @Test public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesFalse() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "false"); + properties.put(CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "false"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, csvFileFormatProperties.isTrimDoubleQuotes()); @@ -130,7 +129,7 @@ public void testAnalyzeFileFormatPropertiesTrimDoubleQuotesFalse() throws Analys @Test public void testAnalyzeFileFormatPropertiesInvalidCompressType() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_COMPRESS_TYPE, "invalid"); + properties.put(CsvFileFormatProperties.PROP_COMPRESS_TYPE, "invalid"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(TFileCompressType.UNKNOWN, csvFileFormatProperties.getCompressionType()); } @@ -138,7 +137,7 @@ public void testAnalyzeFileFormatPropertiesInvalidCompressType() { @Test public void testAnalyzeFileFormatPropertiesValidCompressType() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_COMPRESS_TYPE, "gz"); + properties.put(CsvFileFormatProperties.PROP_COMPRESS_TYPE, "gz"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(TFileCompressType.GZ, csvFileFormatProperties.getCompressionType()); @@ -147,14 +146,14 @@ public void testAnalyzeFileFormatPropertiesValidCompressType() throws AnalysisEx @Test public void testAnalyzeFileFormatPropertiesEmptyCsvSchema() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_CSV_SCHEMA, ""); + properties.put(CsvFileFormatProperties.PROP_CSV_SCHEMA, ""); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); } @Test public void testAnalyzeFileFormatPropertiesValidEncloseMultipleCharacters() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_ENCLOSE, "\"\""); + properties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\"\""); Assert.assertThrows(AnalysisException.class, () -> { csvFileFormatProperties.analyzeFileFormatProperties(properties, true); @@ -164,7 +163,7 @@ public void testAnalyzeFileFormatPropertiesValidEncloseMultipleCharacters() { @Test public void testAnalyzeFileFormatPropertiesValidEncloseEmpty() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_ENCLOSE, ""); + properties.put(CsvFileFormatProperties.PROP_ENCLOSE, ""); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(0, csvFileFormatProperties.getEnclose()); @@ -173,7 +172,7 @@ public void testAnalyzeFileFormatPropertiesValidEncloseEmpty() { @Test public void testAnalyzeFileFormatPropertiesSkipLinesAsString() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_SKIP_LINES, "abc"); + properties.put(CsvFileFormatProperties.PROP_SKIP_LINES, "abc"); Assert.assertThrows(NumberFormatException.class, () -> { csvFileFormatProperties.analyzeFileFormatProperties(properties, true); @@ -183,7 +182,7 @@ public void testAnalyzeFileFormatPropertiesSkipLinesAsString() { @Test public void testAnalyzeFileFormatPropertiesValidColumnSeparator() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_COLUMN_SEPARATOR, ";"); + properties.put(CsvFileFormatProperties.PROP_COLUMN_SEPARATOR, ";"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(";", csvFileFormatProperties.getColumnSeparator()); @@ -192,14 +191,14 @@ public void testAnalyzeFileFormatPropertiesValidColumnSeparator() throws Analysi @Test public void testAnalyzeFileFormatPropertiesLineDelimiterAsString() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_LINE_DELIMITER, "abc"); + properties.put(CsvFileFormatProperties.PROP_LINE_DELIMITER, "abc"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); } @Test public void testAnalyzeFileFormatPropertiesValidLineDelimiter() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_LINE_DELIMITER, "\r\n"); + properties.put(CsvFileFormatProperties.PROP_LINE_DELIMITER, "\r\n"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals("\r\n", csvFileFormatProperties.getLineDelimiter()); @@ -208,7 +207,7 @@ public void testAnalyzeFileFormatPropertiesValidLineDelimiter() throws AnalysisE @Test public void testAnalyzeFileFormatPropertiesValidTrimDoubleQuotes() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); + properties.put(CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "true"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, csvFileFormatProperties.isTrimDoubleQuotes()); @@ -217,7 +216,7 @@ public void testAnalyzeFileFormatPropertiesValidTrimDoubleQuotes() throws Analys @Test public void testAnalyzeFileFormatPropertiesInvalidTrimDoubleQuotes() { Map properties = new HashMap<>(); - properties.put(CsvProperties.PROP_TRIM_DOUBLE_QUOTES, "invalid"); + properties.put(CsvFileFormatProperties.PROP_TRIM_DOUBLE_QUOTES, "invalid"); csvFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, csvFileFormatProperties.isTrimDoubleQuotes()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java index 3e388660dde57d..f614d3223866f1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/JsonFileFormatPropertiesTest.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.property.fileformat; -import org.apache.doris.datasource.property.constants.JsonProperties; import org.apache.doris.nereids.exceptions.AnalysisException; import org.junit.Assert; @@ -52,7 +51,7 @@ public void testAnalyzeFileFormatPropertiesEmpty() throws AnalysisException { @Test public void testAnalyzeFileFormatPropertiesValidJsonRoot() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_JSON_ROOT, "data.items"); + properties.put(JsonFileFormatProperties.PROP_JSON_ROOT, "data.items"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals("data.items", jsonFileFormatProperties.getJsonRoot()); @@ -61,7 +60,7 @@ public void testAnalyzeFileFormatPropertiesValidJsonRoot() throws AnalysisExcept @Test public void testAnalyzeFileFormatPropertiesValidJsonPaths() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_JSON_PATHS, + properties.put(JsonFileFormatProperties.PROP_JSON_PATHS, "[\"$.name\", \"$.age\", \"$.city\"]"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); @@ -71,7 +70,7 @@ public void testAnalyzeFileFormatPropertiesValidJsonPaths() throws AnalysisExcep @Test public void testAnalyzeFileFormatPropertiesStripOuterArrayTrue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_STRIP_OUTER_ARRAY, "true"); + properties.put(JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "true"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, jsonFileFormatProperties.isStripOuterArray()); @@ -80,7 +79,7 @@ public void testAnalyzeFileFormatPropertiesStripOuterArrayTrue() throws Analysis @Test public void testAnalyzeFileFormatPropertiesStripOuterArrayFalse() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_STRIP_OUTER_ARRAY, "false"); + properties.put(JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "false"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, jsonFileFormatProperties.isStripOuterArray()); @@ -89,7 +88,7 @@ public void testAnalyzeFileFormatPropertiesStripOuterArrayFalse() throws Analysi @Test public void testAnalyzeFileFormatPropertiesReadJsonByLineTrue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_READ_JSON_BY_LINE, "true"); + properties.put(JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "true"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, jsonFileFormatProperties.isReadJsonByLine()); @@ -98,7 +97,7 @@ public void testAnalyzeFileFormatPropertiesReadJsonByLineTrue() throws AnalysisE @Test public void testAnalyzeFileFormatPropertiesReadJsonByLineFalse() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_READ_JSON_BY_LINE, "false"); + properties.put(JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "false"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, jsonFileFormatProperties.isReadJsonByLine()); @@ -107,7 +106,7 @@ public void testAnalyzeFileFormatPropertiesReadJsonByLineFalse() throws Analysis @Test public void testAnalyzeFileFormatPropertiesNumAsStringTrue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_NUM_AS_STRING, "true"); + properties.put(JsonFileFormatProperties.PROP_NUM_AS_STRING, "true"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, jsonFileFormatProperties.isNumAsString()); @@ -116,7 +115,7 @@ public void testAnalyzeFileFormatPropertiesNumAsStringTrue() throws AnalysisExce @Test public void testAnalyzeFileFormatPropertiesNumAsStringFalse() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_NUM_AS_STRING, "false"); + properties.put(JsonFileFormatProperties.PROP_NUM_AS_STRING, "false"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, jsonFileFormatProperties.isNumAsString()); @@ -125,7 +124,7 @@ public void testAnalyzeFileFormatPropertiesNumAsStringFalse() throws AnalysisExc @Test public void testAnalyzeFileFormatPropertiesFuzzyParseTrue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_FUZZY_PARSE, "true"); + properties.put(JsonFileFormatProperties.PROP_FUZZY_PARSE, "true"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(true, jsonFileFormatProperties.isFuzzyParse()); @@ -134,7 +133,7 @@ public void testAnalyzeFileFormatPropertiesFuzzyParseTrue() throws AnalysisExcep @Test public void testAnalyzeFileFormatPropertiesFuzzyParseFalse() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_FUZZY_PARSE, "false"); + properties.put(JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, jsonFileFormatProperties.isFuzzyParse()); @@ -143,7 +142,7 @@ public void testAnalyzeFileFormatPropertiesFuzzyParseFalse() throws AnalysisExce @Test public void testAnalyzeFileFormatPropertiesInvalidBooleanValue() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_FUZZY_PARSE, "invalid"); + properties.put(JsonFileFormatProperties.PROP_FUZZY_PARSE, "invalid"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals(false, jsonFileFormatProperties.isFuzzyParse()); @@ -152,12 +151,12 @@ public void testAnalyzeFileFormatPropertiesInvalidBooleanValue() throws Analysis @Test public void testAnalyzeFileFormatPropertiesAllProperties() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_JSON_ROOT, "data.records"); - properties.put(JsonProperties.PROP_JSON_PATHS, "[\"$.id\", \"$.name\"]"); - properties.put(JsonProperties.PROP_STRIP_OUTER_ARRAY, "true"); - properties.put(JsonProperties.PROP_READ_JSON_BY_LINE, "true"); - properties.put(JsonProperties.PROP_NUM_AS_STRING, "true"); - properties.put(JsonProperties.PROP_FUZZY_PARSE, "true"); + properties.put(JsonFileFormatProperties.PROP_JSON_ROOT, "data.records"); + properties.put(JsonFileFormatProperties.PROP_JSON_PATHS, "[\"$.id\", \"$.name\"]"); + properties.put(JsonFileFormatProperties.PROP_STRIP_OUTER_ARRAY, "true"); + properties.put(JsonFileFormatProperties.PROP_READ_JSON_BY_LINE, "true"); + properties.put(JsonFileFormatProperties.PROP_NUM_AS_STRING, "true"); + properties.put(JsonFileFormatProperties.PROP_FUZZY_PARSE, "true"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); @@ -172,7 +171,7 @@ public void testAnalyzeFileFormatPropertiesAllProperties() throws AnalysisExcept @Test public void testAnalyzeFileFormatPropertiesSpecialCharactersInJsonRoot() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_JSON_ROOT, "data.special@#$%^&*()"); + properties.put(JsonFileFormatProperties.PROP_JSON_ROOT, "data.special@#$%^&*()"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals("data.special@#$%^&*()", jsonFileFormatProperties.getJsonRoot()); @@ -181,7 +180,7 @@ public void testAnalyzeFileFormatPropertiesSpecialCharactersInJsonRoot() throws @Test public void testAnalyzeFileFormatPropertiesComplexJsonPaths() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_JSON_PATHS, + properties.put(JsonFileFormatProperties.PROP_JSON_PATHS, "[\"$.deeply.nested[0].array[*].field\", \"$.complex.path[?(@.type=='value')]\"]"); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); @@ -192,7 +191,7 @@ public void testAnalyzeFileFormatPropertiesComplexJsonPaths() throws AnalysisExc @Test public void testAnalyzeFileFormatPropertiesEmptyJsonPaths() throws AnalysisException { Map properties = new HashMap<>(); - properties.put(JsonProperties.PROP_JSON_PATHS, ""); + properties.put(JsonFileFormatProperties.PROP_JSON_PATHS, ""); jsonFileFormatProperties.analyzeFileFormatProperties(properties, true); Assert.assertEquals("", jsonFileFormatProperties.getJsonPaths());