diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java index 5c5d8e39a4b3bb..738a96c07c5a9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -100,6 +101,9 @@ public class JdbcTable extends Table { private boolean connectionPoolKeepAlive; private ExternalFunctionRules functionRules; + // This is used for edit log + @SerializedName("frs") + private String functionRulesString; static { Map tempMap = new CaseInsensitiveMap(); @@ -381,8 +385,9 @@ private void validate(Map properties) throws DdlException { private void checkAndSetExternalFunctionRules(Map properties) throws DdlException { ExternalFunctionRules.check(properties.getOrDefault(JdbcResource.FUNCTION_RULES, "")); - this.functionRules = ExternalFunctionRules.create(jdbcTypeName, - properties.getOrDefault(JdbcResource.FUNCTION_RULES, "")); + String functionRulesString = properties.getOrDefault(JdbcResource.FUNCTION_RULES, ""); + this.functionRules = ExternalFunctionRules.create(jdbcTypeName, functionRulesString); + this.functionRulesString = functionRulesString; } /** @@ -491,4 +496,11 @@ public void setExternalFunctionRules(ExternalFunctionRules functionRules) { public ExternalFunctionRules getExternalFunctionRules() { return functionRules; } + + @Override + public void gsonPostProcess() throws IOException { + super.gsonPostProcess(); + functionRules = ExternalFunctionRules.create(jdbcTypeName, Strings.nullToEmpty(functionRulesString)); + } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 059e12376fb2ee..1fcc7083258910 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -339,6 +339,8 @@ public void afterDropTable(String dbName, String tblName) { if (db.isPresent()) { db.get().setUnInitialized(true); } + LOG.info("after drop table {}.{}.{}. is db present: {}", + dorisCatalog.getName(), dbName, tblName, db.isPresent()); } private void performDropTable(String remoteDbName, String remoteTblName, boolean ifExists) throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java index 2a6a5b201e0258..f03fac85ddb8c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java @@ -248,7 +248,7 @@ public synchronized long write(short op, Writable writable) throws IOException { MetricRepo.COUNTER_CURRENT_EDIT_LOG_SIZE_BYTES.increase((long) theData.getSize()); } if (LOG.isDebugEnabled() || theData.getSize() > (1 << 20)) { - LOG.info("opCode = {}, journal size = {}", op, theData.getSize()); + LOG.info("opCode = {}, journal size = {}, log id: {}", op, theData.getSize(), id); if (MetricRepo.isInit) { MetricRepo.COUNTER_LARGE_EDIT_LOG.increase(1L); } diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out index 2b0147259bbc54..953e628f309658 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out @@ -64,6 +64,9 @@ test_iceberg_case_sensibility_hadoop ICEBERG_HADOOP_CASE_DB2 case_tbl21 k1 1 \N -- !sql18 -- 4 +-- !sql_show -- +case_tbl22 + -- !sql21 -- 5 @@ -159,6 +162,9 @@ test_iceberg_case_sensibility_hadoop ICEBERG_HADOOP_CASE_DB2 case_tbl21 k1 1 \N -- !sql18 -- 5 +-- !sql_show -- +case_tbl22 + -- !sql21 -- 5 @@ -254,6 +260,9 @@ test_iceberg_case_sensibility_hadoop ICEBERG_HADOOP_CASE_DB2 case_tbl21 k1 1 \N -- !sql18 -- 5 +-- !sql_show -- +case_tbl22 + -- !sql21 -- 5 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out index d6d88db3dbed05..c6379666096100 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out @@ -64,6 +64,9 @@ test_iceberg_case_sensibility_rest ICEBERG_REST_CASE_DB2 case_tbl21 k1 1 \N YES -- !sql18 -- 4 +-- !sql_show -- +case_tbl22 + -- !sql21 -- 5 @@ -159,6 +162,9 @@ test_iceberg_case_sensibility_rest ICEBERG_REST_CASE_DB2 case_tbl21 k1 1 \N YES -- !sql18 -- 5 +-- !sql_show -- +case_tbl22 + -- !sql21 -- 5 @@ -254,6 +260,9 @@ test_iceberg_case_sensibility_rest ICEBERG_REST_CASE_DB2 case_tbl21 k1 1 \N YES -- !sql18 -- 5 +-- !sql_show -- +case_tbl22 + -- !sql21 -- 5 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy index ee8ee903e1f9bd..26a35b86cee1a0 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy @@ -278,7 +278,7 @@ suite("test_iceberg_hadoop_case_sensibility", "p0,external,doris,external_docker exception "Table [case_tbl22] does not exist in database [iceberg_hadoop_case_db2]" } sql """create table iceberg_hadoop_case_db2.case_tbl22 (k1 int);""" // recreate - sql """show tables from iceberg_hadoop_case_db2 like "%case_tbl22%"""" + qt_sql_show """show tables from iceberg_hadoop_case_db2 like "%case_tbl22%"""" sql """insert into iceberg_hadoop_case_db2.case_tbl22 values(5);""" order_qt_sql21 """select * from iceberg_hadoop_case_db2.case_tbl22;""" diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy index 22161de615d4a4..510b2c00362a2c 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy @@ -279,7 +279,7 @@ suite("test_iceberg_rest_case_sensibility", "p0,external,doris,external_docker,e exception "Table [case_tbl22] does not exist in database [iceberg_rest_case_db2]" } sql """create table iceberg_rest_case_db2.case_tbl22 (k1 int);""" // recreate - sql """show tables from iceberg_rest_case_db2 like "%case_tbl22%"""" + qt_sql_show """show tables from iceberg_rest_case_db2 like "%case_tbl22%"""" sql """insert into iceberg_rest_case_db2.case_tbl22 values(5);""" order_qt_sql21 """select * from iceberg_rest_case_db2.case_tbl22;""" diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_create_table.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_create_table.groovy index 22364b8bc96c07..e7b2bb9bafcfb1 100644 --- a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_create_table.groovy +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_create_table.groovy @@ -52,9 +52,7 @@ suite("test_iceberg_create_table", "p0,external,doris,external_docker,external_d String tb1 = db1 + "_tb1" String tb2 = db1 + "_tb2" - sql """ drop table if exists ${db1}.${tb1} """ - sql """ drop table if exists ${db1}.${tb2} """ - sql """ drop database if exists ${db1} """ + sql """ drop database if exists ${db1} force""" sql """ create database ${db1} """ diff --git a/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy b/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy index d5d9f3ca032929..b2704a2cf466b9 100644 --- a/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy @@ -333,9 +333,9 @@ suite("test_file_tvf_hdfs","external,hive,tvf,external_docker") { // test create view from tvf and alter view from tvf uri = "${defaultFS}" + "/user/doris/preinstalled_data/csv_format_test/all_types.csv" format = "csv" - sql """ DROP VIEW IF EXISTS test_hdfs_tvf_create_view;""" + sql """ DROP VIEW IF EXISTS test_file_tvf_hdfs_create_view;""" sql """ - create view test_hdfs_tvf_create_view as + create view test_file_tvf_hdfs_create_view as select * from FILE( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", @@ -343,10 +343,10 @@ suite("test_file_tvf_hdfs","external,hive,tvf,external_docker") { "format" = "${format}") order by c1; """ - order_qt_create_view """ select * from test_hdfs_tvf_create_view order by c1 limit 20; """ + order_qt_create_view """ select * from test_file_tvf_hdfs_create_view order by c1 limit 20; """ sql """ - alter view test_hdfs_tvf_create_view as + alter view test_file_tvf_hdfs_create_view as select c1 from FILE( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", @@ -354,7 +354,7 @@ suite("test_file_tvf_hdfs","external,hive,tvf,external_docker") { "format" = "${format}") order by c1; """ - order_qt_alter_view """ select * from test_hdfs_tvf_create_view order by c1 limit 20; """ + order_qt_alter_view """ select * from test_file_tvf_hdfs_create_view order by c1 limit 20; """ } finally { } } diff --git a/regression-test/suites/external_table_p0/tvf/upgrade/load.groovy b/regression-test/suites/external_table_p0/tvf/upgrade/load.groovy index 5fac50e64ffa81..1dd9bbffdd0d3d 100644 --- a/regression-test/suites/external_table_p0/tvf/upgrade/load.groovy +++ b/regression-test/suites/external_table_p0/tvf/upgrade/load.groovy @@ -30,26 +30,26 @@ suite("test_tvf_upgrade_load", "p0,external,hive,external_docker,external_docker // test create view from tvf and alter view from tvf uri = "${defaultFS}" + "/user/doris/preinstalled_data/csv_format_test/all_types.csv" format = "csv" - sql """ DROP VIEW IF EXISTS test_hdfs_tvf_create_view;""" + sql """ DROP VIEW IF EXISTS test_tvf_upgrade_load_create_view;""" sql """ - create view test_hdfs_tvf_create_view as + create view test_tvf_upgrade_load_create_view as select * from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "column_separator" = ",", "format" = "${format}") order by c1; """ - logger.info("View test_hdfs_tvf_create_view created") + logger.info("View test_tvf_upgrade_load_create_view created") sql """ - alter view test_hdfs_tvf_create_view as + alter view test_tvf_upgrade_load_create_view as select c1 from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "column_separator" = ",", "format" = "${format}") order by c1; """ - logger.info("View test_hdfs_tvf_create_view altered") + logger.info("View test_tvf_upgrade_load_create_view altered") } -} \ No newline at end of file +} diff --git a/regression-test/suites/external_table_p0/tvf/upgrade/test.groovy b/regression-test/suites/external_table_p0/tvf/upgrade/test.groovy index a3b0795ef46f98..861fad7ddc8828 100644 --- a/regression-test/suites/external_table_p0/tvf/upgrade/test.groovy +++ b/regression-test/suites/external_table_p0/tvf/upgrade/test.groovy @@ -27,8 +27,8 @@ suite("test_tvf_upgrade_test", "p0,external,hive,external_docker,external_docker String enabled = context.config.otherConfigs.get("enableHiveTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { - order_qt_create_view """ select * from test_hdfs_tvf_create_view order by c1 limit 20; """ + order_qt_create_view """ select * from test_tvf_upgrade_load_create_view order by c1 limit 20; """ - order_qt_alter_view """ select * from test_hdfs_tvf_create_view order by c1 limit 20; """ + order_qt_alter_view """ select * from test_tvf_upgrade_load_create_view order by c1 limit 20; """ } -} \ No newline at end of file +} diff --git a/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy b/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy index f0094c849b3247..8ce441caabd973 100644 --- a/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy @@ -38,8 +38,7 @@ suite("part_partition_invalid", "p0,external,external_docker") { );""" sql """switch ${hive_catalog_name};""" - sql """drop table if exists ${hive_catalog_name}.${hive_database}.${hive_table}""" - sql """ drop database if exists ${hive_database}""" + sql """ drop database if exists ${hive_database} force""" sql """ create database ${hive_database}""" sql """use ${hive_database}""" sql """ @@ -203,9 +202,7 @@ suite("part_partition_invalid", "p0,external,external_docker") { mv_rewrite_success(query_sql, mv_name) order_qt_after_add_data_and_refresh_catalog_and_mv """ ${query_sql}""" - sql """drop table if exists ${hive_catalog_name}.${hive_database}.${hive_table}""" - sql """drop table if exists ${internal_catalog}.${olap_db}.${olap_table}""" - sql """drop database if exists ${hive_catalog_name}.${hive_database}""" + sql """drop database if exists ${hive_catalog_name}.${hive_database} force""" sql """drop materialized view if exists ${internal_catalog}.${olap_db}.${mv_name};""" sql """drop catalog if exists ${hive_catalog_name}""" } diff --git a/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy index 5bd7e5cf338a69..4ecd7065f34a4c 100644 --- a/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy @@ -38,8 +38,7 @@ suite("single_external_table", "p0,external,hive") { );""" sql """switch ${hive_catalog_name};""" - sql """drop table if exists ${hive_catalog_name}.${hive_database}.${hive_table}""" - sql """ drop database if exists ${hive_database}""" + sql """ drop database if exists ${hive_database} force""" sql """ create database ${hive_database}""" sql """use ${hive_database}""" sql """ @@ -192,7 +191,6 @@ suite("single_external_table", "p0,external,hive") { sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5""" - sql """drop table if exists ${hive_catalog_name}.${hive_database}.${hive_table}""" - sql """drop database if exists ${hive_catalog_name}.${hive_database}""" + sql """drop database if exists ${hive_catalog_name}.${hive_database} force""" sql """drop catalog if exists ${hive_catalog_name}""" }