From be3c7ea27b0aa5db9f860a002da47bf7d8e33ea7 Mon Sep 17 00:00:00 2001 From: Kang Date: Sat, 18 Nov 2023 09:49:36 -0600 Subject: [PATCH] Revert "[enhancement](jdbc catalog) Add lowercase column name mapping to Jdbc data source & optimize database and table mapping #27124 (#27130)" (#27230) This reverts commit 087fccd9af38f4003b7cc78bf4cc04af87c561ad. --- .../oracle/init/03-create-table.sql | 7 -- .../docker-compose/oracle/init/04-insert.sql | 2 - docs/en/docs/lakehouse/multi-catalog/jdbc.md | 19 +-- .../docs/lakehouse/multi-catalog/jdbc.md | 35 +++--- .../org/apache/doris/catalog/JdbcTable.java | 41 ++----- .../catalog/external/JdbcExternalTable.java | 2 - .../datasource/jdbc/client/JdbcClient.java | 108 +++++++----------- .../jdbc/client/JdbcMySQLClient.java | 23 ++-- .../jdbc/client/JdbcOracleClient.java | 25 ++-- .../planner/external/jdbc/JdbcScanNode.java | 16 +-- .../planner/external/odbc/OdbcScanNode.java | 2 +- .../jdbc/test_oracle_jdbc_catalog.out | 9 -- .../jdbc/test_oracle_jdbc_catalog.groovy | 3 - 13 files changed, 96 insertions(+), 196 deletions(-) diff --git a/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql b/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql index 24bf6a7a856e98..046f18c0dc65fc 100644 --- a/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql @@ -132,13 +132,6 @@ age number(2), score number(3,1) ); -CREATE TABLE "DORIS_TEST"."student3" -( -"id" NUMBER(5,0), -"NAME" VARCHAR2(20), -"AGE" NUMBER(2,0), -"SCORE" NUMBER(3,1) -); create table doris_test.test_all_types ( id int, diff --git a/docker/thirdparties/docker-compose/oracle/init/04-insert.sql b/docker/thirdparties/docker-compose/oracle/init/04-insert.sql index 663851cfa4ec95..888cdf5bf7e4e4 100644 --- a/docker/thirdparties/docker-compose/oracle/init/04-insert.sql +++ b/docker/thirdparties/docker-compose/oracle/init/04-insert.sql @@ -86,8 +86,6 @@ insert into doris_test."student2" values (2, 'bob', 21, 90.5); insert into doris_test."student2" values (3, 'jerry', 23, 88.0); insert into doris_test."student2" values (4, 'andy', 21, 93); -insert into doris_test."student3" values(1, 'doris', 3, 1.0); - insert into doris_test.test_all_types values (1, 111, 123, 7456123.89, 573, 34, 673.43, 34.1264, 56.2, 23.231, 99, 9999, 999999999, 999999999999999999, 999, 99999, 9999999999, 9999999999999999999, diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md b/docs/en/docs/lakehouse/multi-catalog/jdbc.md index 5fa29b130650e7..13151bc2f8db5b 100644 --- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md @@ -52,7 +52,7 @@ PROPERTIES ("key"="value", ...) | `driver_url ` | Yes | | JDBC Driver Jar | | `driver_class ` | Yes | | JDBC Driver Class | | `only_specified_database` | No | "false" | Whether only the database specified to be synchronized. | -| `lower_case_table_names` | No | "false" | Whether to synchronize the database name, table name and column name of jdbc external data source in lowercase. | +| `lower_case_table_names` | No | "false" | Whether to synchronize jdbc external data source table names in lower case. | | `include_database_list` | No | "" | When only_specified_database=true，only synchronize the specified databases. split with ','. db name is case sensitive. | | `exclude_database_list` | No | "" | When only_specified_database=true，do not synchronize the specified databases. split with ','. db name is case sensitive. | @@ -68,7 +68,7 @@ PROPERTIES ("key"="value", ...) ### Lowercase table name synchronization -When `lower_case_table_names` is set to `true`, Doris is able to query non-lowercase databases and tables and columns by maintaining a mapping of lowercase names to actual names on the remote system +When `lower_case_table_names` is set to `true`, Doris is able to query non-lowercase databases and tables by maintaining a mapping of lowercase names to actual names on the remote system **Notice:** @@ -78,9 +78,9 @@ When `lower_case_table_names` is set to `true`, Doris is able to query non-lower For other databases, you still need to specify the real library name and table name when querying. -2. In Doris 2.0.3 and later versions, it is valid for all databases. When querying, all database names and table names and columns will be converted into real names and then queried. If you upgrade from an old version to 2.0. 3, `Refresh ` is required to take effect. +2. In Doris 2.0.3 and later versions, it is valid for all databases. When querying, all library names and table names will be converted into real names and then queried. If you upgrade from an old version to 2.0. 3, `Refresh ` is required to take effect. - However, if the database or table or column names differ only in case, such as `Doris` and `doris`, Doris cannot query them due to ambiguity. + However, if the database or table names differ only in case, such as `Doris` and `doris`, Doris cannot query them due to ambiguity. 3. When the FE parameter's `lower_case_table_names` is set to `1` or `2`, the JDBC Catalog's `lower_case_table_names` parameter must be set to `true`. If the FE parameter's `lower_case_table_names` is set to `0`, the JDBC Catalog parameter can be `true` or `false` and defaults to `false`. This ensures consistency and predictability in how Doris handles internal and external table configurations. @@ -113,8 +113,8 @@ In some cases, the keywords in the database might be used as the field names. Fo ### Predicate Pushdown 1. When executing a query like `where dt = '2022-01-01'`, Doris can push down these filtering conditions to the external data source, thereby directly excluding data that does not meet the conditions at the data source level, reducing the number of unqualified Necessary data acquisition and transfer. This greatly improves query performance while also reducing the load on external data sources. - -2. When `enable_func_pushdown` is set to true, the function conditions after where will also be pushed down to the external data source. Currently, only MySQL and ClickHouse are supported. If you encounter a function that is not supported by MySQL or ClickHouse, you can set this parameter to false. , currently Doris will automatically identify some functions not supported by MySQL and functions supported by CLickHouse for push-down condition filtering, which can be viewed through explain sql. + +2. When `enable_func_pushdown` is set to true, the function condition after where will also be pushed down to the external data source. Currently, only MySQL is supported. If you encounter a function that MySQL does not support, you can set this parameter to false, at present, Doris will automatically identify some functions not supported by MySQL to filter the push-down conditions, which can be checked by explain sql. Functions that are currently not pushed down include: @@ -123,13 +123,6 @@ Functions that are currently not pushed down include: | DATE_TRUNC | | MONEY_FORMAT | -Functions that are currently pushed down include: - -| ClickHouse | -|:--------------:| -| FROM_UNIXTIME | -| UNIX_TIMESTAMP | - ### Line Limit If there is a limit keyword in the query, Doris will translate it into semantics suitable for different data sources. diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md index acc057caa54208..545f5ada6e23e6 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md @@ -44,16 +44,16 @@ PROPERTIES ("key"="value", ...) ## 参数说明 -| 参数 | 必须 | 默认值 | 说明 | -|---------------------------|-----|---------|-----------------------------------------------------------------------| -| `user` | 是 | | 对应数据库的用户名 | -| `password` | 是 | | 对应数据库的密码 | -| `jdbc_url` | 是 | | JDBC 连接串 | -| `driver_url` | 是 | | JDBC Driver Jar 包名称 | -| `driver_class` | 是 | | JDBC Driver Class 名称 | -| `lower_case_table_names` | 否 | "false" | 是否以小写的形式同步jdbc外部数据源的库名和表名以及列名 | -| `only_specified_database` | 否 | "false" | 指定是否只同步指定的 database | -| `include_database_list` | 否 | "" | 当only_specified_database=true时，指定同步多个database，以','分隔。db名称是大小写敏感的。 | +| 参数 | 必须 | 默认值 | 说明 | +|---------------------------|-----|---------|---------------------------------------------------------------------------------------------| +| `user` | 是 | | 对应数据库的用户名 | +| `password` | 是 | | 对应数据库的密码 | +| `jdbc_url` | 是 | | JDBC 连接串 | +| `driver_url` | 是 | | JDBC Driver Jar 包名称 | +| `driver_class` | 是 | | JDBC Driver Class 名称 | +| `lower_case_table_names` | 否 | "false" | 是否以小写的形式同步jdbc外部数据源的库名和表名 | +| `only_specified_database` | 否 | "false" | 指定是否只同步指定的 database | +| `include_database_list` | 否 | "" | 当only_specified_database=true时，指定同步多个database，以','分隔。db名称是大小写敏感的。 | | `exclude_database_list` | 否 | "" | 当only_specified_database=true时，指定不需要同步的多个database，以','分割。db名称是大小写敏感的。 | ### 驱动包路径 @@ -68,7 +68,7 @@ PROPERTIES ("key"="value", ...) ### 小写表名同步 -当 `lower_case_table_names` 设置为 `true` 时，Doris 通过维护小写名称到远程系统中实际名称的映射，能够查询非小写的数据库和表以及列 +当 `lower_case_table_names` 设置为 `true` 时，Doris 通过维护小写名称到远程系统中实际名称的映射，能够查询非小写的数据库和表 **注意：** @@ -78,9 +78,9 @@ PROPERTIES ("key"="value", ...) 对于其他数据库，仍需要在查询时指定真实的库名和表名。 -2. 在 Doris 2.0.3 及之后的版本，对所有的数据库都有效，在查询时，会将所有的库名和表名以及列名转换为真实的名称，再去查询，如果是从老版本升级到 2.0.3 ，需要 `Refresh ` 才能生效。 +2. 在 Doris 2.0.3 及之后的版本，对所有的数据库都有效，在查询时，会将所有的库名和表名转换为真实的名称，再去查询，如果是从老版本升级到 2.0.3 ，需要 `Refresh ` 才能生效。 - 但是，如果库名、表名或列名只有大小写不同，例如 `Doris` 和 `doris`，则 Doris 由于歧义而无法查询它们。 + 但是，如果数据库或者表名只有大小写不同，例如 `Doris` 和 `doris`，则 Doris 由于歧义而无法查询它们。 3. 当 FE 参数的 `lower_case_table_names` 设置为 `1` 或 `2` 时，JDBC Catalog 的 `lower_case_table_names` 参数必须设置为 `true`。如果 FE 参数的 `lower_case_table_names` 设置为 `0`，则 JDBC Catalog 的参数可以为 `true` 或 `false`，默认为 `false`。这确保了 Doris 在处理内部和外部表配置时的一致性和可预测性。 @@ -114,7 +114,7 @@ select * from mysql_catalog.mysql_database.mysql_table where k1 > 1000 and k3 =' 1. 当执行类似于 `where dt = '2022-01-01'` 这样的查询时，Doris 能够将这些过滤条件下推到外部数据源，从而直接在数据源层面排除不符合条件的数据，减少了不必要的数据获取和传输。这大大提高了查询性能，同时也降低了对外部数据源的负载。 -2. 当 `enable_func_pushdown` 设置为true，会将 where 之后的函数条件也下推到外部数据源，目前仅支持 MySQL 以及 ClickHouse，如遇到 MySQL 或 ClickHouse 不支持的函数，可以将此参数设置为 false，目前 Doris 会自动识别部分 MySQL 不支持的函数以及 CLickHouse 支持的函数进行下推条件过滤，可通过 explain sql 查看。 +2. 当 `enable_func_pushdown` 设置为true，会将 where 之后的函数条件也下推到外部数据源，目前仅支持 MySQL，如遇到 MySQL 不支持的函数，可以将此参数设置为 false，目前 Doris 会自动识别部分 MySQL 不支持的函数进行下推条件过滤，可通过 explain sql 查看。 目前不会下推的函数有： @@ -123,13 +123,6 @@ select * from mysql_catalog.mysql_database.mysql_table where k1 > 1000 and k3 =' | DATE_TRUNC | | MONEY_FORMAT | -目前会下推的函数有： - -| ClickHouse | -|:--------------:| -| FROM_UNIXTIME | -| UNIX_TIMESTAMP | - ### 行数限制 如果在查询中带有 limit 关键字，Doris 会将其转译成适合不同数据源的语义。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java index fe397929acc4cb..795f00e7bfed9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java @@ -27,8 +27,6 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.collect.Maps; import lombok.Setter; @@ -49,12 +47,9 @@ public class JdbcTable extends Table { private static final Logger LOG = LogManager.getLogger(JdbcTable.class); - private static final ObjectMapper objectMapper = new ObjectMapper(); - private static final String TABLE = "table"; private static final String REAL_DATABASE = "real_database"; private static final String REAL_TABLE = "real_table"; - private static final String REAL_COLUMNS = "real_columns"; private static final String RESOURCE = "resource"; private static final String TABLE_TYPE = "table_type"; private static final String URL = "jdbc_url"; @@ -70,7 +65,6 @@ public class JdbcTable extends Table { // real name only for jdbc catalog private String realDatabaseName; private String realTableName; - private Map realColumnNames; private String jdbcTypeName; @@ -116,7 +110,7 @@ public String getInsertSql(List insertCols) { sb.append(getProperRealFullTableName(TABLE_TYPE_MAP.get(getTableTypeName()))); sb.append("("); List transformedInsertCols = insertCols.stream() - .map(col -> getProperRealColumnName(TABLE_TYPE_MAP.get(getTableTypeName()), col)) + .map(col -> databaseProperName(TABLE_TYPE_MAP.get(getTableTypeName()), col)) .collect(Collectors.toList()); sb.append(String.join(",", transformedInsertCols)); sb.append(")"); @@ -206,7 +200,6 @@ public void write(DataOutput out) throws IOException { serializeMap.put(CHECK_SUM, checkSum); serializeMap.put(REAL_DATABASE, realDatabaseName); serializeMap.put(REAL_TABLE, realTableName); - serializeMap.put(REAL_COLUMNS, objectMapper.writeValueAsString(realColumnNames)); int size = (int) serializeMap.values().stream().filter(v -> { return v != null; @@ -243,9 +236,6 @@ public void readFields(DataInput in) throws IOException { checkSum = serializeMap.get(CHECK_SUM); realDatabaseName = serializeMap.get(REAL_DATABASE); realTableName = serializeMap.get(REAL_TABLE); - String realColumnNamesJson = serializeMap.get(REAL_COLUMNS); - realColumnNames = objectMapper.readValue(realColumnNamesJson, new TypeReference>() { - }); } public String getResourceName() { @@ -273,14 +263,6 @@ public String getProperRealFullTableName(TOdbcTableType tableType) { } } - public String getProperRealColumnName(TOdbcTableType tableType, String columnName) { - if (realColumnNames == null || realColumnNames.isEmpty() || !realColumnNames.containsKey(columnName)) { - return databaseProperName(tableType, columnName); - } else { - return properNameWithRealName(tableType, realColumnNames.get(columnName)); - } - } - public String getTableTypeName() { return jdbcTypeName; } @@ -376,13 +358,14 @@ private void validate(Map properties) throws DdlException { * @param wrapEnd The character(s) to be added at the end of each name component. * @param toUpperCase If true, convert the name to upper case. * @param toLowerCase If true, convert the name to lower case. - *

- * Note: If both toUpperCase and toLowerCase are true, the name will ultimately be converted to lower case. - *

- * The name is expected to be in the format of 'schemaName.tableName'. If there is no '.', - * the function will treat the entire string as one name component. - * If there is a '.', the function will treat the string before the first '.' as the schema name - * and the string after the '.' as the table name. + *

+ * Note: If both toUpperCase and toLowerCase are true, the name will ultimately be converted to lower case. + *

+ * The name is expected to be in the format of 'schemaName.tableName'. If there is no '.', + * the function will treat the entire string as one name component. + * If there is a '.', the function will treat the string before the first '.' as the schema name + * and the string after the '.' as the table name. + * * @return The formatted name. */ public static String formatName(String name, String wrapStart, String wrapEnd, boolean toUpperCase, @@ -403,18 +386,18 @@ public static String formatName(String name, String wrapStart, String wrapEnd, b /** * Formats a database name according to the database type. - *

+ * * Rules: * - MYSQL, OCEANBASE: Wrap with backticks (`), case unchanged. Example: mySchema.myTable -> `mySchema.myTable` * - SQLSERVER: Wrap with square brackets ([]), case unchanged. Example: mySchema.myTable -> [mySchema].[myTable] * - POSTGRESQL, CLICKHOUSE, TRINO, OCEANBASE_ORACLE, SAP_HANA: Wrap with double quotes ("), case unchanged. - * Example: mySchema.myTable -> "mySchema"."myTable" + * Example: mySchema.myTable -> "mySchema"."myTable" * - ORACLE: Wrap with double quotes ("), convert to upper case. Example: mySchema.myTable -> "MYSCHEMA"."MYTABLE" * For other types, the name is returned as is. * * @param tableType The database type. * @param name The name to be formatted, expected in 'schemaName.tableName' format. If no '.', treats entire string - * as one name component. If '.', treats string before first '.' as schema name and after as table name. + * as one name component. If '.', treats string before first '.' as schema name and after as table name. * @return The formatted name. */ public static String databaseProperName(TOdbcTableType tableType, String name) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java index 9c5159a2bd8083..a02c59080fc4eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java @@ -91,8 +91,6 @@ private JdbcTable toJdbcTable() { jdbcTable.setRealDatabaseName(((JdbcExternalCatalog) catalog).getJdbcClient().getRealDatabaseName(this.dbName)); jdbcTable.setRealTableName( ((JdbcExternalCatalog) catalog).getJdbcClient().getRealTableName(this.dbName, this.name)); - jdbcTable.setRealColumnNames(((JdbcExternalCatalog) catalog).getJdbcClient().getRealColumnNames(this.dbName, - this.name)); jdbcTable.setJdbcTypeName(jdbcCatalog.getDatabaseTypeName()); jdbcTable.setJdbcUrl(jdbcCatalog.getJdbcUrl()); jdbcTable.setJdbcUser(jdbcCatalog.getJdbcUser()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index b4e2604bfc9880..97052caccd6e2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -67,15 +67,10 @@ public abstract class JdbcClient { // only used when isLowerCaseTableNames = true. protected final ConcurrentHashMap lowerDBToRealDB = new ConcurrentHashMap<>(); // only used when isLowerCaseTableNames = true. - protected final ConcurrentHashMap> lowerTableToRealTable - = new ConcurrentHashMap<>(); - // only used when isLowerCaseTableNames = true. - protected final ConcurrentHashMap>> - lowerColumnToRealColumn = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap lowerTableToRealTable = new ConcurrentHashMap<>(); private final AtomicBoolean dbNamesLoaded = new AtomicBoolean(false); private final AtomicBoolean tableNamesLoaded = new AtomicBoolean(false); - private final AtomicBoolean columnNamesLoaded = new AtomicBoolean(false); public static JdbcClient createJdbcClient(JdbcClientConfig jdbcClientConfig) { String dbType = parseDbType(jdbcClientConfig.getJdbcUrl()); @@ -183,7 +178,7 @@ public void close(AutoCloseable... closeables) { if (closeable != null) { try { closeable.close(); - } catch (Exception e) { + } catch (Exception e) { throw new JdbcClientException("Can not close : ", e); } } @@ -191,10 +186,8 @@ public void close(AutoCloseable... closeables) { } // This part used to process meta-information of database, table and column. - /** * get all database name through JDBC - * * @return list of database names */ public List getDatabaseNameList() { @@ -215,8 +208,6 @@ public List getDatabaseNameList() { if (isLowerCaseTableNames) { lowerDBToRealDB.put(databaseName.toLowerCase(), databaseName); databaseName = databaseName.toLowerCase(); - } else { - lowerDBToRealDB.put(databaseName, databaseName); } tempDatabaseNames.add(databaseName); } @@ -246,20 +237,20 @@ public List getDatabaseNameList() { * get all tables of one database */ public List getTablesNameList(String dbName) { + String currentDbName = dbName; List tablesName = Lists.newArrayList(); String[] tableTypes = getTableTypes(); - String finalDbName = getRealDatabaseName(dbName); + if (isLowerCaseTableNames) { + currentDbName = getRealDatabaseName(dbName); + } + String finalDbName = currentDbName; processTable(finalDbName, null, tableTypes, (rs) -> { try { while (rs.next()) { String tableName = rs.getString("TABLE_NAME"); if (isLowerCaseTableNames) { - lowerTableToRealTable.putIfAbsent(finalDbName, new ConcurrentHashMap<>()); - lowerTableToRealTable.get(finalDbName).put(tableName.toLowerCase(), tableName); + lowerTableToRealTable.put(tableName.toLowerCase(), tableName); tableName = tableName.toLowerCase(); - } else { - lowerTableToRealTable.putIfAbsent(finalDbName, new ConcurrentHashMap<>()); - lowerTableToRealTable.get(finalDbName).put(tableName, tableName); } tablesName.add(tableName); } @@ -271,10 +262,16 @@ public List getTablesNameList(String dbName) { } public boolean isTableExist(String dbName, String tableName) { + String currentDbName = dbName; + String currentTableName = tableName; final boolean[] isExist = {false}; + if (isLowerCaseTableNames) { + currentDbName = getRealDatabaseName(dbName); + currentTableName = getRealTableName(dbName, tableName); + } String[] tableTypes = getTableTypes(); - String finalDbName = getRealDatabaseName(dbName); - String finalTableName = getRealTableName(dbName, tableName); + String finalTableName = currentTableName; + String finalDbName = currentDbName; processTable(finalDbName, finalTableName, tableTypes, (rs) -> { try { if (rs.next()) { @@ -295,25 +292,23 @@ public List getJdbcColumnsInfo(String dbName, String tableName) Connection conn = getConnection(); ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String finalDbName = getRealDatabaseName(dbName); - String finalTableName = getRealTableName(dbName, tableName); + // if isLowerCaseTableNames == true, tableName is lower case + // but databaseMetaData.getColumns() is case sensitive + String currentDbName = dbName; + String currentTableName = tableName; + if (isLowerCaseTableNames) { + currentDbName = getRealDatabaseName(dbName); + currentTableName = getRealTableName(dbName, tableName); + } + String finalDbName = currentDbName; + String finalTableName = currentTableName; try { DatabaseMetaData databaseMetaData = conn.getMetaData(); String catalogName = getCatalogName(conn); rs = getColumns(databaseMetaData, catalogName, finalDbName, finalTableName); while (rs.next()) { - lowerColumnToRealColumn.putIfAbsent(finalDbName, new ConcurrentHashMap<>()); - lowerColumnToRealColumn.get(finalDbName).putIfAbsent(finalTableName, new ConcurrentHashMap<>()); JdbcFieldSchema field = new JdbcFieldSchema(); - String columnName = rs.getString("COLUMN_NAME"); - if (isLowerCaseTableNames) { - lowerColumnToRealColumn.get(finalDbName).get(finalTableName) - .put(columnName.toLowerCase(), columnName); - columnName = columnName.toLowerCase(); - } else { - lowerColumnToRealColumn.get(finalDbName).get(finalTableName).put(columnName, columnName); - } - field.setColumnName(columnName); + field.setColumnName(rs.getString("COLUMN_NAME")); field.setDataType(rs.getInt("DATA_TYPE")); field.setDataTypeName(rs.getString("TYPE_NAME")); /* @@ -357,9 +352,11 @@ public List getColumnsFromJdbc(String dbName, String tableName) { } public String getRealDatabaseName(String dbname) { - if (lowerDBToRealDB == null - || lowerDBToRealDB.isEmpty() - || !lowerDBToRealDB.containsKey(dbname)) { + if (!isLowerCaseTableNames) { + return dbname; + } + + if (lowerDBToRealDB.isEmpty() || !lowerDBToRealDB.containsKey(dbname)) { loadDatabaseNamesIfNeeded(); } @@ -367,34 +364,15 @@ public String getRealDatabaseName(String dbname) { } public String getRealTableName(String dbName, String tableName) { - String realDbName = getRealDatabaseName(dbName); - if (lowerTableToRealTable == null - || lowerTableToRealTable.isEmpty() - || !lowerTableToRealTable.containsKey(realDbName) - || lowerTableToRealTable.get(realDbName) == null - || lowerTableToRealTable.get(realDbName).isEmpty() - || !lowerTableToRealTable.get(realDbName).containsKey(tableName) - || lowerTableToRealTable.get(realDbName).get(tableName) == null) { - loadTableNamesIfNeeded(dbName); + if (!isLowerCaseTableNames) { + return tableName; } - return lowerTableToRealTable.get(realDbName).get(tableName); - } - - public Map getRealColumnNames(String dbName, String tableName) { - String realDbName = getRealDatabaseName(dbName); - String realTableName = getRealTableName(dbName, tableName); - if (lowerColumnToRealColumn == null - || lowerColumnToRealColumn.isEmpty() - || !lowerColumnToRealColumn.containsKey(realDbName) - || lowerColumnToRealColumn.get(realDbName) == null - || lowerColumnToRealColumn.get(realDbName).isEmpty() - || !lowerColumnToRealColumn.get(realDbName).containsKey(realTableName) - || lowerColumnToRealColumn.get(realDbName).get(realTableName) == null - || lowerColumnToRealColumn.get(realDbName).get(realTableName).isEmpty()) { - loadColumnNamesIfNeeded(dbName, tableName); + if (lowerTableToRealTable.isEmpty() || !lowerTableToRealTable.containsKey(tableName)) { + loadTableNamesIfNeeded(dbName); } - return lowerColumnToRealColumn.get(realDbName).get(realTableName); + + return lowerTableToRealTable.get(tableName); } private void loadDatabaseNamesIfNeeded() { @@ -409,12 +387,6 @@ private void loadTableNamesIfNeeded(String dbName) { } } - private void loadColumnNamesIfNeeded(String dbName, String tableName) { - if (columnNamesLoaded.compareAndSet(false, true)) { - getJdbcColumnsInfo(dbName, tableName); - } - } - // protected methods,for subclass to override protected String getCatalogName(Connection conn) throws SQLException { return null; @@ -439,7 +411,7 @@ protected String[] getTableTypes() { } protected void processTable(String dbName, String tableName, String[] tableTypes, - Consumer resultSetConsumer) { + Consumer resultSetConsumer) { Connection conn = getConnection(); ResultSet rs = null; try { @@ -463,7 +435,7 @@ protected boolean isTableModified(String modifiedTableName, String actualTableNa } protected ResultSet getColumns(DatabaseMetaData databaseMetaData, String catalogName, String schemaName, - String tableName) throws SQLException { + String tableName) throws SQLException { return databaseMetaData.getColumns(catalogName, schemaName, tableName, null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index 3273f4b7591c21..61ba2a0db47d07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -35,7 +35,6 @@ import java.sql.Statement; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; public class JdbcMySQLClient extends JdbcClient { @@ -118,27 +117,21 @@ public List getJdbcColumnsInfo(String dbName, String tableName) Connection conn = getConnection(); ResultSet rs = null; List tableSchema = com.google.common.collect.Lists.newArrayList(); - String finalDbName = getRealDatabaseName(dbName); - String finalTableName = getRealTableName(dbName, tableName); + // if isLowerCaseTableNames == true, tableName is lower case + // but databaseMetaData.getColumns() is case sensitive + if (isLowerCaseTableNames) { + dbName = lowerDBToRealDB.get(dbName); + tableName = lowerTableToRealTable.get(tableName); + } try { DatabaseMetaData databaseMetaData = conn.getMetaData(); String catalogName = getCatalogName(conn); - rs = getColumns(databaseMetaData, catalogName, finalDbName, finalTableName); + rs = getColumns(databaseMetaData, catalogName, dbName, tableName); List primaryKeys = getPrimaryKeys(databaseMetaData, catalogName, dbName, tableName); Map mapFieldtoType = null; while (rs.next()) { - lowerColumnToRealColumn.putIfAbsent(finalDbName, new ConcurrentHashMap<>()); - lowerColumnToRealColumn.get(finalDbName).putIfAbsent(finalTableName, new ConcurrentHashMap<>()); JdbcFieldSchema field = new JdbcFieldSchema(); - String columnName = rs.getString("COLUMN_NAME"); - if (isLowerCaseTableNames) { - lowerColumnToRealColumn.get(finalDbName).get(finalTableName) - .put(columnName.toLowerCase(), columnName); - columnName = columnName.toLowerCase(); - } else { - lowerColumnToRealColumn.get(finalDbName).get(finalTableName).put(columnName, columnName); - } - field.setColumnName(columnName); + field.setColumnName(rs.getString("COLUMN_NAME")); field.setDataType(rs.getInt("DATA_TYPE")); // in mysql-jdbc-connector-8.0.*, TYPE_NAME of the HLL column in doris will be "UNKNOWN" diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index 270b5b4bdc8b2f..d0a9f2c3de7ab3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -28,7 +28,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; public class JdbcOracleClient extends JdbcClient { @@ -62,8 +61,6 @@ public List getDatabaseNameList() { if (isLowerCaseTableNames) { lowerDBToRealDB.put(databaseName.toLowerCase(), databaseName); databaseName = databaseName.toLowerCase(); - } else { - lowerDBToRealDB.put(databaseName, databaseName); } tempDatabaseNames.add(databaseName); } @@ -94,8 +91,14 @@ public List getJdbcColumnsInfo(String dbName, String tableName) Connection conn = getConnection(); ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String finalDbName = getRealDatabaseName(dbName); - String finalTableName = getRealTableName(dbName, tableName); + String currentDbName = dbName; + String currentTableName = tableName; + if (isLowerCaseTableNames) { + currentDbName = getRealDatabaseName(dbName); + currentTableName = getRealTableName(dbName, tableName); + } + String finalDbName = currentDbName; + String finalTableName = currentTableName; try { DatabaseMetaData databaseMetaData = conn.getMetaData(); String catalogName = getCatalogName(conn); @@ -116,18 +119,8 @@ public List getJdbcColumnsInfo(String dbName, String tableName) if (isModify && isTableModified(rs.getString("TABLE_NAME"), finalTableName)) { continue; } - lowerColumnToRealColumn.putIfAbsent(finalDbName, new ConcurrentHashMap<>()); - lowerColumnToRealColumn.get(finalDbName).putIfAbsent(finalTableName, new ConcurrentHashMap<>()); JdbcFieldSchema field = new JdbcFieldSchema(); - String columnName = rs.getString("COLUMN_NAME"); - if (isLowerCaseTableNames) { - lowerColumnToRealColumn.get(finalDbName).get(finalTableName) - .put(columnName.toLowerCase(), columnName); - columnName = columnName.toLowerCase(); - } else { - lowerColumnToRealColumn.get(finalDbName).get(finalTableName).put(columnName, columnName); - } - field.setColumnName(columnName); + field.setColumnName(rs.getString("COLUMN_NAME")); field.setDataType(rs.getInt("DATA_TYPE")); field.setDataTypeName(rs.getString("TYPE_NAME")); /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java index 4eccaff47741ab..23a44ed643263e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java @@ -138,7 +138,7 @@ private void createJdbcFilters() { List pushDownConjuncts = collectConjunctsToPushDown(conjunctsList, errors); for (Expr individualConjunct : pushDownConjuncts) { - String filter = conjunctExprToString(jdbcType, individualConjunct, tbl); + String filter = conjunctExprToString(jdbcType, individualConjunct); filters.add(filter); conjuncts.remove(individualConjunct); } @@ -169,9 +169,9 @@ private void createJdbcColumns() { continue; } Column col = slot.getColumn(); - columns.add(tbl.getProperRealColumnName(jdbcType, col.getName())); + columns.add(JdbcTable.databaseProperName(jdbcType, col.getName())); } - if (columns.isEmpty()) { + if (0 == columns.size()) { columns.add("*"); } } @@ -324,12 +324,12 @@ private static boolean containsFunctionCallExpr(Expr expr) { return !fnExprList.isEmpty(); } - public static String conjunctExprToString(TOdbcTableType tableType, Expr expr, JdbcTable tbl) { + public static String conjunctExprToString(TOdbcTableType tableType, Expr expr) { if (expr instanceof CompoundPredicate) { StringBuilder result = new StringBuilder(); CompoundPredicate compoundPredicate = (CompoundPredicate) expr; for (Expr child : compoundPredicate.getChildren()) { - result.append(conjunctExprToString(tableType, child, tbl)); + result.append(conjunctExprToString(tableType, child)); result.append(" ").append(compoundPredicate.getOp().toString()).append(" "); } // Remove the last operator @@ -357,11 +357,7 @@ public static String conjunctExprToString(TOdbcTableType tableType, Expr expr, J ArrayList children = expr.getChildren(); String filter; if (children.get(0) instanceof SlotRef) { - if (tbl != null) { - filter = tbl.getProperRealColumnName(tableType, children.get(0).toMySql()); - } else { - filter = JdbcTable.databaseProperName(tableType, children.get(0).toMySql()); - } + filter = JdbcTable.databaseProperName(tableType, children.get(0).toMySql()); } else { filter = children.get(0).toMySql(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java index 68950ebab833cc..bf4e835e4f1035 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java @@ -184,7 +184,7 @@ private void createOdbcFilters(Analyzer analyzer) { ArrayList odbcConjuncts = Expr.cloneList(conjuncts, sMap); for (Expr p : odbcConjuncts) { if (shouldPushDownConjunct(odbcType, p)) { - String filter = JdbcScanNode.conjunctExprToString(odbcType, p, null); + String filter = JdbcScanNode.conjunctExprToString(odbcType, p); filters.add(filter); conjuncts.remove(p); } diff --git a/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out index 0545ddafac5c00..519c7ab131cd98 100644 --- a/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out @@ -187,15 +187,6 @@ DORIS_TEST 3 jerry 23 88.0 4 andy 21 93.0 --- !lower_case_column_names1 -- -1 doris 3 1.0 - --- !lower_case_column_names2 -- -1 doris 3 1.0 - --- !lower_case_column_names3 -- -1 doris 3 1.0 - -- !query_clob -- 10010 liantong 10086 yidong diff --git a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy index f3463763b87289..e762f2f4aaf41f 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy @@ -214,9 +214,6 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d // test lower case name order_qt_lower_case_table_names4 """ select * from student2 order by id; """ - order_qt_lower_case_column_names1 """ select * from student3 order by id; """ - order_qt_lower_case_column_names2 """ select * from student3 where id = 1 order by id; """ - order_qt_lower_case_column_names3 """ select * from student3 where id = 1 and name = 'doris' order by id; """ sql """drop catalog if exists ${catalog_name} """